View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.tools.hbase;
19  
20  import java.io.IOException;
21  
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.hbase.HBaseConfiguration;
24  import org.apache.hadoop.hbase.HColumnDescriptor;
25  import org.apache.hadoop.hbase.HTableDescriptor;
26  import org.apache.hadoop.hbase.TableName;
27  import org.apache.hadoop.hbase.client.Admin;
28  import org.apache.hadoop.hbase.client.Connection;
29  import org.apache.hadoop.hbase.client.ConnectionFactory;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.omid.HBaseShims;
32  import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
33  import org.apache.omid.committable.hbase.KeyGenerator;
34  import org.apache.omid.committable.hbase.KeyGeneratorImplementations;
35  import org.apache.omid.committable.hbase.RegionSplitter;
36  import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
37  import org.slf4j.Logger;
38  import org.slf4j.LoggerFactory;
39  
40  import com.beust.jcommander.IParameterValidator;
41  import com.beust.jcommander.JCommander;
42  import com.beust.jcommander.Parameter;
43  import com.beust.jcommander.ParameterException;
44  import com.beust.jcommander.Parameters;
45  import com.beust.jcommander.ParametersDelegate;
46  
47  /**
48   * Helper class to create required HBase tables by Omid
49   */
50  public class OmidTableManager {
51  
52      private static final Logger LOG = LoggerFactory.getLogger(OmidTableManager.class);
53  
54      public static final String COMMIT_TABLE_COMMAND_NAME = "commit-table";
55      static final String TIMESTAMP_TABLE_COMMAND_NAME = "timestamp-table";
56  
57      private static final byte[][] commitTableFamilies = new byte[][]{
58              Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME),
59              Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME)};
60      private static final byte[][] timestampTableFamilies = new byte[][]{
61              Bytes.toBytes(HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME)};
62  
63      private JCommander commandLine;
64      private MainConfig mainConfig = new MainConfig();
65      private CommitTableCommand commitTableCommand = new CommitTableCommand();
66      private TimestampTableCommand timestampTableCommand = new TimestampTableCommand();
67  
68      public OmidTableManager(String... args) {
69          commandLine = new JCommander(mainConfig);
70          commandLine.addCommand(COMMIT_TABLE_COMMAND_NAME, commitTableCommand);
71          commandLine.addCommand(TIMESTAMP_TABLE_COMMAND_NAME, timestampTableCommand);
72          try {
73              commandLine.parse(args);
74          } catch (ParameterException ex) {
75              commandLine.usage();
76              throw new IllegalArgumentException(ex.getMessage());
77          }
78      }
79  
80      public void executeActionsOnHBase(Configuration hbaseConf) throws IOException {
81  
82          HBaseLogin.loginIfNeeded(mainConfig.loginFlags);
83  
84          try (Connection conn = ConnectionFactory.createConnection(hbaseConf);
85               Admin hBaseAdmin = conn.getAdmin()) {
86              byte[][] tableFamilies;
87              byte[][] splitKeys = new byte[0][0];
88              String tableName;
89  
90              LOG.info("----------------------------------------------------------------------------------------------");
91              switch (commandLine.getParsedCommand()) {
92                  case COMMIT_TABLE_COMMAND_NAME:
93                      LOG.info("Performing actions related to COMMIT TABLE");
94                      tableName = commitTableCommand.tableName;
95                      tableFamilies = commitTableFamilies;
96                      if (commitTableCommand.numRegions > 1) {
97                          splitKeys = splitInUniformRegions(hbaseConf, commitTableCommand.numRegions);
98                      }
99                      break;
100                 case TIMESTAMP_TABLE_COMMAND_NAME:
101                     LOG.info("Performing actions related to TIMESTAMP TABLE");
102                     tableName = timestampTableCommand.tableName;
103                     tableFamilies = timestampTableFamilies;
104                     break;
105                 default:
106                     LOG.error("Unknown command: {}", commandLine.getParsedCommand());
107                     commandLine.usage();
108                     return;
109             }
110 
111             createTable(hBaseAdmin, tableName, tableFamilies, splitKeys, 1);
112             LOG.info("----------------------------------------------------------------------------------------------");
113 
114         }
115     }
116 
117     public static void main(String... args) throws Exception {
118 
119         OmidTableManager tableManager = new OmidTableManager(args);
120         tableManager.executeActionsOnHBase(HBaseConfiguration.create());
121 
122     }
123 
124     // ----------------------------------------------------------------------------------------------------------------
125     // Helper methods and classes
126     // ----------------------------------------------------------------------------------------------------------------
127 
128     private static byte[][] splitInUniformRegions(Configuration hBaseConf, int numRegions) throws IOException {
129 
130         KeyGenerator keyGen = KeyGeneratorImplementations.defaultKeyGenerator();
131         RegionSplitter.SplitAlgorithm algo =
132                 RegionSplitter.newSplitAlgoInstance(hBaseConf, RegionSplitter.UniformSplit.class.getName());
133         algo.setFirstRow(algo.rowToStr(keyGen.startTimestampToKey(0)));
134         algo.setLastRow(algo.rowToStr(keyGen.startTimestampToKey(Long.MAX_VALUE)));
135 
136         // Return the split keys
137         return algo.split(numRegions);
138 
139     }
140 
141     private static void createTable(Admin admin, String tableName, byte[][] families, byte[][] splitKeys,
142                                     int maxVersions)
143             throws IOException {
144 
145         LOG.info("About to create Table named {} with {} splits", tableName, splitKeys.length);
146 
147         TableName hTableName = TableName.valueOf(tableName); 
148         if (admin.tableExists(hTableName)) {
149             LOG.error("Table {} already exists. Table creation cancelled", tableName);
150             return;
151         }
152 
153         HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
154 
155         for (byte[] family : families) {
156             HColumnDescriptor colDescriptor = new HColumnDescriptor(family);
157             colDescriptor.setMaxVersions(maxVersions);
158             HBaseShims.addFamilyToHTableDescriptor(tableDescriptor, colDescriptor);
159             LOG.info("\tAdding Family {}", colDescriptor);
160         }
161 
162         admin.createTable(tableDescriptor, splitKeys);
163 
164         LOG.info("Table {} created. Regions: {}", tableName, admin.getTableRegions(hTableName).size());
165 
166     }
167 
168     // Configuration-related classes
169 
170     static class MainConfig {
171 
172         @ParametersDelegate
173         SecureHBaseConfig loginFlags = new SecureHBaseConfig();
174 
175     }
176 
177     @Parameters(commandDescription = "Specifies configuration for the Commit Table")
178     static class CommitTableCommand {
179 
180         @Parameter(names = "-tableName", description = "Table name where to stores the commits", required = false)
181         String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
182 
183         @Parameter(names = "-numRegions", description = "Number of splits (to pre-split tableName)", required = false,
184                    validateWith = IntegerGreaterThanZero.class)
185         int numRegions = 16;
186 
187     }
188 
189     @Parameters(commandDescription = "Specifies configuration for the Timestamp Table")
190     static class TimestampTableCommand {
191 
192         @Parameter(names = "-tableName", description = "Table name where to store timestamps")
193         String tableName = HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
194 
195     }
196 
197     public static class IntegerGreaterThanZero implements IParameterValidator {
198 
199         @Override
200         public void validate(String name, String value) throws ParameterException {
201             int n = Integer.parseInt(value);
202             if (n <= 0) {
203                 throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")");
204             }
205         }
206 
207     }
208 
209 }