View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.omid.transaction;
19  
20  import com.google.common.base.Optional;
21  import org.apache.hadoop.hbase.Cell;
22  import org.apache.hadoop.hbase.KeyValue;
23  import org.apache.hadoop.hbase.KeyValue.Type;
24  import org.apache.hadoop.hbase.util.Bytes;
25  import org.apache.omid.HBaseShims;
26  import org.testng.annotations.DataProvider;
27  import org.testng.annotations.Test;
28  
29  import java.io.IOException;
30  import java.util.ArrayList;
31  import java.util.List;
32  import java.util.SortedMap;
33  
34  import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
35  import static org.testng.Assert.assertEquals;
36  import static org.testng.Assert.assertFalse;
37  import static org.testng.Assert.assertTrue;
38  import static org.testng.Assert.fail;
39  
40  @Test(groups = "noHBase")
41  public class TestCellUtils {
42  
43      private final byte[] row = Bytes.toBytes("test-row");
44      private final byte[] family = Bytes.toBytes("test-family");
45      private final byte[] qualifier = Bytes.toBytes("test-qual");
46      private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
47  
48      @DataProvider(name = "shadow-cell-suffixes")
49      public Object[][] createShadowCellSuffixes() {
50          return new Object[][]{
51                  {SHADOW_CELL_SUFFIX},
52          };
53      }
54  
55      @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
56      public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
57  
58          final byte[] validShadowCellQualifier =
59                  com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
60          final byte[] sandwichValidShadowCellQualifier =
61                  com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
62          final byte[] doubleEndedValidShadowCellQualifier =
63                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
64          final byte[] interleavedValidShadowCellQualifier =
65                  com.google.common.primitives.Bytes.concat(validShadowCellQualifier, com.google.common.primitives.Bytes
66                          .concat(validShadowCellQualifier, validShadowCellQualifier));
67          final byte[] value = Bytes.toBytes("test-value");
68  
69          // Test the qualifier passed is a shadow cell
70          // qualifier because it contains only one suffix
71          // and is placed at the end of the qualifier:
72          // qual_nameSUFFIX
73          KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
74          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
75  
76          // We also accept this pattern in the qualifier:
77          // SUFFIXqual_nameSUFFIX
78          kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
79          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
80  
81          // We also accept this pattern in the qualifier:
82          // qual_nameSUFFIXSUFFIX
83          kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
84          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
85  
86          // We also accept this pattern in the qualifier:
87          // qual_nameSUFFIXqual_nameSUFFIXqual_nameSUFFIX
88          kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
89          assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
90  
91          // Test the qualifier passed is not a shadow cell
92          // qualifier if there's nothing else apart from the suffix
93          kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
94          assertFalse(CellUtils.isShadowCell(kv), "Should not include a valid shadowCell identifier");
95  
96      }
97  
98      @Test(timeOut = 10_000)
99      public void testCorrectMapingOfCellsToShadowCells() throws IOException {
100         // Create the required data
101         final byte[] validShadowCellQualifier =
102                 com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
103 
104         final byte[] qualifier2 = Bytes.toBytes("test-qual2");
105         final byte[] validShadowCellQualifier2 =
106                 com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);
107 
108         final byte[] qualifier3 = Bytes.toBytes("test-qual3");
109 
110         Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
111         Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
112         Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
113         Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
114         Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
115 
116         Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
117         Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
118 
119         Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
120 
121         // Check a list of cells with duplicate values
122         List<Cell> badListWithDups = new ArrayList<>();
123         badListWithDups.add(cell1);
124         badListWithDups.add(dupCell1WithAnotherValue);
125 
126         // Check dup shadow cell with same MVCC is ignored
127         SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
128         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
129         assertTrue(cellsToShadowCells.containsKey(cell1));
130         KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
131         KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
132         assertTrue(firstKey.equals(lastKey));
133         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
134                                         cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()),
135                    "Should be equal");
136 
137         // Modify dup shadow cell to have a greater MVCC and check that is replaced
138         HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
139         cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
140         assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
141         assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
142         firstKey = (KeyValue) cellsToShadowCells.firstKey();
143         lastKey = (KeyValue) cellsToShadowCells.lastKey();
144         assertTrue(firstKey.equals(lastKey));
145         assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(),
146                                         firstKey.getValueLength(), dupCell1WithAnotherValue.getValueArray(),
147                                         dupCell1WithAnotherValue.getValueOffset(), dupCell1WithAnotherValue.getValueLength()),
148                    "Should be equal");
149         // Check a list of cells with duplicate values
150         List<Cell> cellListWithDups = new ArrayList<>();
151         cellListWithDups.add(cell1);
152         cellListWithDups.add(shadowCell1);
153         cellListWithDups.add(dupCell1); // Dup cell
154         cellListWithDups.add(delCell1); // Another Dup cell but with different type
155         cellListWithDups.add(cell2);
156         cellListWithDups.add(cell3);
157         cellListWithDups.add(shadowCell2);
158 
159         cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
160         assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
161         assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
162         assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
163         assertFalse(cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
164         // The current algo avoids to put the delete cell
165         // as key after the put cell with same value was added
166         assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
167         assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
168 
169     }
170 
171     @Test(timeOut = 10_000)
172     public void testShadowCellSuffixConcatenationToQualifier() {
173 
174         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
175         byte[] suffixedQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
176                                                                  cell.getQualifierOffset(),
177                                                                  cell.getQualifierLength());
178         byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
179         assertEquals(suffixedQualifier, expectedQualifier);
180 
181     }
182 
183     @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
184     public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
185 
186         // Test removal from a correclty suffixed qualifier
187         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
188         Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
189         byte[] resultedQualifier = CellUtils.removeShadowCellSuffix(cell.getQualifierArray(),
190                                                                     cell.getQualifierOffset(),
191                                                                     cell.getQualifierLength());
192         byte[] expectedQualifier = qualifier;
193         assertEquals(resultedQualifier, expectedQualifier);
194 
195         // Test removal from a badly suffixed qualifier
196         byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD"));
197         Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
198         try {
199             CellUtils.removeShadowCellSuffix(badCell.getQualifierArray(),
200                                              badCell.getQualifierOffset(),
201                                              badCell.getQualifierLength());
202             fail();
203         } catch (IllegalArgumentException e) {
204             // Expected
205         }
206     }
207 
208     @Test(timeOut = 10_000)
209     public void testMatchingQualifiers() {
210         Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
211         assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
212         assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
213     }
214 
215     @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
216     public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
217         // Test suffixed qualifier
218         byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
219         int originalQualifierLength =
220                 CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
221         assertEquals(originalQualifierLength, qualifier.length);
222 
223         // Test passing qualifier without shadow cell suffix
224         originalQualifierLength =
225                 CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
226         assertEquals(originalQualifierLength, qualifier.length);
227     }
228 
229 }