View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.tools.hbase;
19  
20  import com.beust.jcommander.IParameterValidator;
21  import com.beust.jcommander.JCommander;
22  import com.beust.jcommander.Parameter;
23  import com.beust.jcommander.ParameterException;
24  import com.beust.jcommander.Parameters;
25  import com.beust.jcommander.ParametersDelegate;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.HBaseConfiguration;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.client.HBaseAdmin;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.omid.HBaseShims;
34  import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
35  import org.apache.omid.committable.hbase.KeyGenerator;
36  import org.apache.omid.committable.hbase.KeyGeneratorImplementations;
37  import org.apache.omid.committable.hbase.RegionSplitter;
38  import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
39  import org.slf4j.Logger;
40  import org.slf4j.LoggerFactory;
41  
42  import java.io.IOException;
43  
44  /**
45   * Helper class to create required HBase tables by Omid
46   */
47  public class OmidTableManager {
48  
49      private static final Logger LOG = LoggerFactory.getLogger(OmidTableManager.class);
50  
51      public static final String COMMIT_TABLE_COMMAND_NAME = "commit-table";
52      static final String TIMESTAMP_TABLE_COMMAND_NAME = "timestamp-table";
53  
54      private static final byte[][] commitTableFamilies = new byte[][]{
55              Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME),
56              Bytes.toBytes(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME)};
57      private static final byte[][] timestampTableFamilies = new byte[][]{
58              Bytes.toBytes(HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME)};
59  
60      private JCommander commandLine;
61      private MainConfig mainConfig = new MainConfig();
62      private CommitTableCommand commitTableCommand = new CommitTableCommand();
63      private TimestampTableCommand timestampTableCommand = new TimestampTableCommand();
64  
65      public OmidTableManager(String... args) {
66          commandLine = new JCommander(mainConfig);
67          commandLine.addCommand(COMMIT_TABLE_COMMAND_NAME, commitTableCommand);
68          commandLine.addCommand(TIMESTAMP_TABLE_COMMAND_NAME, timestampTableCommand);
69          try {
70              commandLine.parse(args);
71          } catch (ParameterException ex) {
72              commandLine.usage();
73              throw new IllegalArgumentException(ex.getMessage());
74          }
75      }
76  
77      public void executeActionsOnHBase(Configuration hbaseConf) throws IOException {
78  
79          HBaseLogin.loginIfNeeded(mainConfig.loginFlags);
80  
81          try (HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConf)) {
82              byte[][] tableFamilies;
83              byte[][] splitKeys = new byte[0][0];
84              String tableName;
85  
86              LOG.info("----------------------------------------------------------------------------------------------");
87              switch (commandLine.getParsedCommand()) {
88                  case COMMIT_TABLE_COMMAND_NAME:
89                      LOG.info("Performing actions related to COMMIT TABLE");
90                      tableName = commitTableCommand.tableName;
91                      tableFamilies = commitTableFamilies;
92                      if (commitTableCommand.numRegions > 1) {
93                          splitKeys = splitInUniformRegions(hbaseConf, commitTableCommand.numRegions);
94                      }
95                      break;
96                  case TIMESTAMP_TABLE_COMMAND_NAME:
97                      LOG.info("Performing actions related to TIMESTAMP TABLE");
98                      tableName = timestampTableCommand.tableName;
99                      tableFamilies = timestampTableFamilies;
100                     break;
101                 default:
102                     LOG.error("Unknown command: {}", commandLine.getParsedCommand());
103                     commandLine.usage();
104                     return;
105             }
106 
107             createTable(hBaseAdmin, tableName, tableFamilies, splitKeys, 1);
108             LOG.info("----------------------------------------------------------------------------------------------");
109 
110         }
111     }
112 
113     public static void main(String... args) throws Exception {
114 
115         OmidTableManager tableManager = new OmidTableManager(args);
116         tableManager.executeActionsOnHBase(HBaseConfiguration.create());
117 
118     }
119 
120     // ----------------------------------------------------------------------------------------------------------------
121     // Helper methods and classes
122     // ----------------------------------------------------------------------------------------------------------------
123 
124     private static byte[][] splitInUniformRegions(Configuration hBaseConf, int numRegions) throws IOException {
125 
126         KeyGenerator keyGen = KeyGeneratorImplementations.defaultKeyGenerator();
127         RegionSplitter.SplitAlgorithm algo =
128                 RegionSplitter.newSplitAlgoInstance(hBaseConf, RegionSplitter.UniformSplit.class.getName());
129         algo.setFirstRow(algo.rowToStr(keyGen.startTimestampToKey(0)));
130         algo.setLastRow(algo.rowToStr(keyGen.startTimestampToKey(Long.MAX_VALUE)));
131 
132         // Return the split keys
133         return algo.split(numRegions);
134 
135     }
136 
137     private static void createTable(HBaseAdmin admin, String tableName, byte[][] families, byte[][] splitKeys,
138                                     int maxVersions)
139             throws IOException {
140 
141         LOG.info("About to create Table named {} with {} splits", tableName, splitKeys.length);
142 
143         if (admin.tableExists(tableName)) {
144             LOG.error("Table {} already exists. Table creation cancelled", tableName);
145             return;
146         }
147 
148         HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
149 
150         for (byte[] family : families) {
151             HColumnDescriptor colDescriptor = new HColumnDescriptor(family);
152             colDescriptor.setMaxVersions(maxVersions);
153             HBaseShims.addFamilyToHTableDescriptor(tableDescriptor, colDescriptor);
154             LOG.info("\tAdding Family {}", colDescriptor);
155         }
156 
157         admin.createTable(tableDescriptor, splitKeys);
158 
159         LOG.info("Table {} created. Regions: {}", tableName, admin.getTableRegions(Bytes.toBytes(tableName)).size());
160 
161     }
162 
163     // Configuration-related classes
164 
165     static class MainConfig {
166 
167         @ParametersDelegate
168         SecureHBaseConfig loginFlags = new SecureHBaseConfig();
169 
170     }
171 
172     @Parameters(commandDescription = "Specifies configuration for the Commit Table")
173     static class CommitTableCommand {
174 
175         @Parameter(names = "-tableName", description = "Table name where to stores the commits", required = false)
176         String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
177 
178         @Parameter(names = "-numRegions", description = "Number of splits (to pre-split tableName)", required = false,
179                    validateWith = IntegerGreaterThanZero.class)
180         int numRegions = 16;
181 
182     }
183 
184     @Parameters(commandDescription = "Specifies configuration for the Timestamp Table")
185     static class TimestampTableCommand {
186 
187         @Parameter(names = "-tableName", description = "Table name where to store timestamps")
188         String tableName = HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
189 
190     }
191 
192     public static class IntegerGreaterThanZero implements IParameterValidator {
193 
194         public void validate(String name, String value) throws ParameterException {
195             int n = Integer.parseInt(value);
196             if (n <= 0) {
197                 throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")");
198             }
199         }
200 
201     }
202 
203 }