View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.transaction;
19  
20  import com.google.common.base.Optional;
21  import org.apache.hadoop.hbase.Cell;
22  import org.apache.hadoop.hbase.KeyValue;
23  import org.apache.hadoop.hbase.KeyValue.Type;
24  import org.apache.hadoop.hbase.util.Bytes;
25  import org.apache.omid.HBaseShims;
26  import org.testng.annotations.DataProvider;
27  import org.testng.annotations.Test;
28  
29  import java.io.IOException;
30  import java.util.ArrayList;
31  import java.util.List;
32  import java.util.SortedMap;
33  
34  import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
35  import static org.testng.Assert.assertEquals;
36  import static org.testng.Assert.assertFalse;
37  import static org.testng.Assert.assertTrue;
38  import static org.testng.Assert.fail;
39  
40  @Test(groups = "noHBase")
41  public class TestCellUtils {
42  
43      private final byte[] row = Bytes.toBytes("test-row");
44      private final byte[] family = Bytes.toBytes("test-family");
45      private final byte[] qualifier = Bytes.toBytes("test-qual");
46      private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
47  
48      @DataProvider(name = "shadow-cell-suffixes")
49      public Object[][] createShadowCellSuffixes() {
50          return new Object[][]{
51                  {SHADOW_CELL_SUFFIX},
52          };
53      }
54  
55      @Test(dataProvider = "shadow-cell-suffixes")
56      public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
57  
58          final byte[] validShadowCellQualifier =
59                  com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
60          final byte[] sandwichValidShadowCellQualifier =
61                  com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
62          final byte[] doubleEndedValidShadowCellQualifier =
63                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
64          final byte[] interleavedValidShadowCellQualifier =
65                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier,
66                          com.google.common.primitives.Bytes
67                                  .concat(validShadowCellQualifier, validShadowCellQualifier));
68          final byte[] value = Bytes.toBytes("test-value");
69  
70          // Test the qualifier passed is a shadow cell
71          // qualifier because it contains only one suffix
72          // and is placed at the end of the qualifier:
73          // qual_nameSUFFIX
74          KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
75          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
76  
77          // We also accept this pattern in the qualifier:
78          // SUFFIXqual_nameSUFFIX
79          kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
80          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
81  
82          // We also accept this pattern in the qualifier:
83          // qual_nameSUFFIXSUFFIX
84          kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
85          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
86  
87          // We also accept this pattern in the qualifier:
88          // qual_nameSUFFIXqual_nameSUFFIXqual_nameSUFFIX
89          kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
90          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
91  
92          // Test the qualifier passed is not a shadow cell
93          // qualifier if there's nothing else apart from the suffix
94          kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
95          assertFalse(CellUtils.isShadowCell(kv), "Should not include a valid shadowCell identifier");
96  
97      }
98  
99      @Test
100     public void testCorrectMapingOfCellsToShadowCells() throws IOException {
101         // Create the required data
102         final byte[] validShadowCellQualifier =
103                 com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
104 
105         final byte[] qualifier2 = Bytes.toBytes("test-qual2");
106         final byte[] validShadowCellQualifier2 =
107                 com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);
108 
109         final byte[] qualifier3 = Bytes.toBytes("test-qual3");
110 
111         Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
112         Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
113         Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
114         Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
115         Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
116 
117         Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
118         Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
119 
120         Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
121 
122         // Check a list of cells with duplicate values
123         List<Cell> badListWithDups = new ArrayList<>();
124         badListWithDups.add(cell1);
125         badListWithDups.add(dupCell1WithAnotherValue);
126 
127         // Check dup shadow cell with same MVCC is ignored
128         SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
129         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
130         assertTrue(cellsToShadowCells.containsKey(cell1));
131         KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
132         KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
133         assertTrue(firstKey.equals(lastKey));
134         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
135                                         cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()),
136                    "Should be equal");
137 
138         // Modify dup shadow cell to have a greater MVCC and check that is replaced
139         HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
140         cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
141         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
142         assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
143         firstKey = (KeyValue) cellsToShadowCells.firstKey();
144         lastKey = (KeyValue) cellsToShadowCells.lastKey();
145         assertTrue(firstKey.equals(lastKey));
146         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(),
147                                         firstKey.getValueLength(), dupCell1WithAnotherValue.getValueArray(),
148                                         dupCell1WithAnotherValue.getValueOffset(), dupCell1WithAnotherValue.getValueLength()),
149                    "Should be equal");
150         // Check a list of cells with duplicate values
151         List<Cell> cellListWithDups = new ArrayList<>();
152         cellListWithDups.add(cell1);
153         cellListWithDups.add(shadowCell1);
154         cellListWithDups.add(dupCell1); // Dup cell
155         cellListWithDups.add(delCell1); // Another Dup cell but with different type
156         cellListWithDups.add(cell2);
157         cellListWithDups.add(cell3);
158         cellListWithDups.add(shadowCell2);
159 
160         cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
161         assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
162         assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
163         assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
164         assertFalse(cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
165         // The current algo avoids to put the delete cell
166         // as key after the put cell with same value was added
167         assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
168         assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
169 
170     }
171 
172     @Test
173     public void testShadowCellSuffixConcatenationToQualifier() {
174 
175         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
176         byte[] suffixedQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
177                 cell.getQualifierOffset(),
178                 cell.getQualifierLength());
179         byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
180         assertEquals(suffixedQualifier, expectedQualifier);
181 
182     }
183 
184     @Test(dataProvider = "shadow-cell-suffixes")
185     public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
186 
187         // Test removal from a correclty suffixed qualifier
188         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
189         Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
190         byte[] resultedQualifier = CellUtils.removeShadowCellSuffix(cell.getQualifierArray(),
191                 cell.getQualifierOffset(),
192                 cell.getQualifierLength());
193         byte[] expectedQualifier = qualifier;
194         assertEquals(resultedQualifier, expectedQualifier);
195 
196         // Test removal from a badly suffixed qualifier
197         byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD"));
198         Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
199         try {
200             CellUtils.removeShadowCellSuffix(badCell.getQualifierArray(),
201                     badCell.getQualifierOffset(),
202                     badCell.getQualifierLength());
203             fail();
204         } catch (IllegalArgumentException e) {
205             // Expected
206         }
207     }
208 
209     @Test
210     public void testMatchingQualifiers() {
211         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
212         assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
213         assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
214     }
215 
216     @Test(dataProvider = "shadow-cell-suffixes")
217     public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
218         // Test suffixed qualifier
219         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
220         int originalQualifierLength =
221                 CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
222         assertEquals(originalQualifierLength, qualifier.length);
223 
224         // Test passing qualifier without shadow cell suffix
225         originalQualifierLength =
226                 CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
227         assertEquals(originalQualifierLength, qualifier.length);
228     }
229 
230 }