View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.transaction;
19  
20  import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
21  import org.apache.hadoop.hbase.Cell;
22  import org.apache.hadoop.hbase.KeyValue;
23  import org.apache.hadoop.hbase.KeyValue.Type;
24  import org.apache.hadoop.hbase.util.Bytes;
25  import org.apache.omid.HBaseShims;
26  import org.apache.omid.tso.client.CellId;
27  import org.testng.annotations.DataProvider;
28  import org.testng.annotations.Test;
29  
30  import java.io.IOException;
31  import java.util.ArrayList;
32  import java.util.List;
33  import java.util.SortedMap;
34  
35  import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_PREFIX;
36  import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
37  import static org.testng.Assert.assertEquals;
38  import static org.testng.Assert.assertFalse;
39  import static org.testng.Assert.assertTrue;
40  import static org.testng.Assert.fail;
41  
42  @Test(groups = "noHBase")
43  public class TestCellUtils {
44  
45      private final byte[] row = Bytes.toBytes("test-row");
46      private final byte[] family = Bytes.toBytes("test-family");
47      private final byte[] qualifier = Bytes.toBytes("test-qual");
48      private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
49  
50      @DataProvider(name = "shadow-cell-suffixes")
51      public Object[][] createShadowCellSuffixes() {
52          return new Object[][]{
53                  {SHADOW_CELL_SUFFIX},
54          };
55      }
56  
57      @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
58      public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
59  
60          final byte[] validShadowCellQualifier =
61                  com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
62          final byte[] sandwichValidShadowCellQualifier =
63                  com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
64          final byte[] doubleEndedValidShadowCellQualifier =
65                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
66          final byte[] interleavedValidShadowCellQualifier =
67                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier, com.google.common.primitives.Bytes
68                          .concat(validShadowCellQualifier, validShadowCellQualifier));
69          final byte[] value = Bytes.toBytes("test-value");
70  
71          // Test the qualifier passed is a shadow cell
72          // qualifier because it contains only one suffix
73          // and is placed at the end of the qualifier:
74          // qual_nameSUFFIX
75          KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
76          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
77  
78          // We also accept this pattern in the qualifier:
79          // SUFFIXqual_nameSUFFIX
80          kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
81          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
82  
83          // We also accept this pattern in the qualifier:
84          // qual_nameSUFFIXSUFFIX
85          kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
86          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
87  
88          // We also accept this pattern in the qualifier:
89          // qual_nameSUFFIXqual_nameSUFFIXqual_nameSUFFIX
90          kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
91          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
92  
93          // Test the qualifier passed is not a shadow cell
94          // qualifier if there's nothing else apart from the suffix
95          kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
96          assertFalse(CellUtils.isShadowCell(kv), "Should not include a valid shadowCell identifier");
97  
98      }
99  
100     @Test(timeOut = 10_000)
101     public void testCorrectMapingOfCellsToShadowCells() throws IOException {
102         // Create the required data
103         final byte[] validShadowCellQualifier =
104                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
105 
106         final byte[] qualifier2 = Bytes.toBytes("test-qual2");
107         final byte[] validShadowCellQualifier2 =
108                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier2, SHADOW_CELL_SUFFIX);
109 
110         final byte[] qualifier3 = Bytes.toBytes("test-qual3");
111 
112         Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
113         Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
114         Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
115         Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
116         Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
117 
118         Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
119         Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
120 
121         Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
122 
123         // Check a list of cells with duplicate values
124         List<Cell> badListWithDups = new ArrayList<>();
125         badListWithDups.add(cell1);
126         badListWithDups.add(dupCell1WithAnotherValue);
127 
128         // Check dup shadow cell with same MVCC is ignored
129         SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
130         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
131         assertTrue(cellsToShadowCells.containsKey(cell1));
132         KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
133         KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
134         assertTrue(firstKey.equals(lastKey));
135         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
136                                         cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()),
137                    "Should be equal");
138 
139         // Modify dup shadow cell to have a greater MVCC and check that is replaced
140         HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
141         cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
142         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
143         assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
144         firstKey = (KeyValue) cellsToShadowCells.firstKey();
145         lastKey = (KeyValue) cellsToShadowCells.lastKey();
146         assertTrue(firstKey.equals(lastKey));
147         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(),
148                                         firstKey.getValueLength(), dupCell1WithAnotherValue.getValueArray(),
149                                         dupCell1WithAnotherValue.getValueOffset(), dupCell1WithAnotherValue.getValueLength()),
150                    "Should be equal");
151         // Check a list of cells with duplicate values
152         List<Cell> cellListWithDups = new ArrayList<>();
153         cellListWithDups.add(cell1);
154         cellListWithDups.add(shadowCell1);
155         cellListWithDups.add(dupCell1); // Dup cell
156         cellListWithDups.add(delCell1); // Another Dup cell but with different type
157         cellListWithDups.add(cell2);
158         cellListWithDups.add(cell3);
159         cellListWithDups.add(shadowCell2);
160 
161         cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
162         assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
163         assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
164         assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
165         assertFalse(cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
166         // The current algo avoids to put the delete cell
167         // as key after the put cell with same value was added
168         assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
169         assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
170 
171     }
172 
173     @Test(timeOut = 10_000)
174     public void testShadowCellSuffixConcatenationToQualifier() {
175 
176         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
177         byte[] suffixedQualifier = CellUtils.addShadowCellSuffixPrefix(cell.getQualifierArray(),
178                                                                  cell.getQualifierOffset(),
179                                                                  cell.getQualifierLength());
180         byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
181         assertEquals(suffixedQualifier, expectedQualifier);
182 
183     }
184 
185     @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
186     public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
187 
188         // Test removal from a correclty suffixed qualifier
189         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, shadowCellSuffixToTest);
190         Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
191         byte[] resultedQualifier = CellUtils.removeShadowCellSuffixPrefix(cell.getQualifierArray(),
192                                                                     cell.getQualifierOffset(),
193                                                                     cell.getQualifierLength());
194         byte[] expectedQualifier = qualifier;
195         assertEquals(resultedQualifier, expectedQualifier);
196 
197         // Test removal from a badly suffixed qualifier
198         byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, Bytes.toBytes("BAD"));
199         Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
200         try {
201             CellUtils.removeShadowCellSuffixPrefix(badCell.getQualifierArray(),
202                                              badCell.getQualifierOffset(),
203                                              badCell.getQualifierLength());
204             fail();
205         } catch (IllegalArgumentException e) {
206             // Expected
207         }
208     }
209 
210     @Test(timeOut = 10_000)
211     public void testMatchingQualifiers() {
212         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
213         assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
214         assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
215     }
216 
217     @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
218     public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
219         // Test suffixed qualifier
220         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, shadowCellSuffixToTest);
221         int originalQualifierLength =
222                 CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
223         assertEquals(originalQualifierLength, qualifier.length);
224 
225         // Test passing qualifier without shadow cell suffix
226         originalQualifierLength =
227                 CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
228         assertEquals(originalQualifierLength, qualifier.length);
229     }
230 
231 
232     @Test(timeOut = 10_000)
233     public void testmapCellsToShadowCellsCellOrder() {
234         // Create the required data
235         final byte[] validShadowCellQualifier =
236                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
237 
238         final byte[] qualifier2 = Bytes.toBytes("test-qual2");
239         final byte[] validShadowCellQualifier2 =
240                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier2, SHADOW_CELL_SUFFIX);
241 
242         final byte[] qualifier3 = Bytes.toBytes("test-qual3");
243         final byte[] validShadowCellQualifier3 =
244                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier3, SHADOW_CELL_SUFFIX);
245 
246         final byte[] qualifier4 = Bytes.toBytes("test-qual4");
247         final byte[] qualifier5 = Bytes.toBytes("test-qual5");
248         final byte[] validShadowCellQualifier5 =
249                 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier5, SHADOW_CELL_SUFFIX);
250 
251 
252         Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
253         Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
254 
255         Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
256         Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
257 
258         Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
259         Cell shadowCell3 = new KeyValue(row, family, validShadowCellQualifier3, 1, Bytes.toBytes("sc-value2"));
260 
261         Cell cell4 = new KeyValue(row, family, qualifier4, 1, Bytes.toBytes("value4"));
262 
263         Cell shadowCell5 = new KeyValue(row, family, validShadowCellQualifier5, 1, Bytes.toBytes("sc-value2"));
264 
265         List<Cell> scanList = new ArrayList<>();
266         scanList.add(shadowCell5);
267         scanList.add(cell3);
268         scanList.add(cell1);
269         scanList.add(shadowCell1);
270         scanList.add(shadowCell2);
271         scanList.add(cell4);
272         scanList.add(cell2);
273         scanList.add(shadowCell3);
274         scanList.add(shadowCell5);
275 
276         SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(scanList);
277         assertEquals(cellsToShadowCells.get(cell1).get(), shadowCell1);
278         assertEquals(cellsToShadowCells.get(cell2).get(), shadowCell2);
279         assertEquals(cellsToShadowCells.get(cell3).get(), shadowCell3);
280         assertFalse(cellsToShadowCells.get(cell4).isPresent());
281     }
282 
283 }