1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.omid.transaction;
19
20 import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
21 import org.apache.hadoop.hbase.Cell;
22 import org.apache.hadoop.hbase.KeyValue;
23 import org.apache.hadoop.hbase.KeyValue.Type;
24 import org.apache.hadoop.hbase.util.Bytes;
25 import org.apache.omid.HBaseShims;
26 import org.apache.omid.tso.client.CellId;
27 import org.testng.annotations.DataProvider;
28 import org.testng.annotations.Test;
29
30 import java.io.IOException;
31 import java.util.ArrayList;
32 import java.util.List;
33 import java.util.SortedMap;
34
35 import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_PREFIX;
36 import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
37 import static org.testng.Assert.assertEquals;
38 import static org.testng.Assert.assertFalse;
39 import static org.testng.Assert.assertTrue;
40 import static org.testng.Assert.fail;
41
42 @Test(groups = "noHBase")
43 public class TestCellUtils {
44
45 private final byte[] row = Bytes.toBytes("test-row");
46 private final byte[] family = Bytes.toBytes("test-family");
47 private final byte[] qualifier = Bytes.toBytes("test-qual");
48 private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
49
50 @DataProvider(name = "shadow-cell-suffixes")
51 public Object[][] createShadowCellSuffixes() {
52 return new Object[][]{
53 {SHADOW_CELL_SUFFIX},
54 };
55 }
56
57 @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
58 public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
59
60 final byte[] validShadowCellQualifier =
61 com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
62 final byte[] sandwichValidShadowCellQualifier =
63 com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
64 final byte[] doubleEndedValidShadowCellQualifier =
65 com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
66 final byte[] interleavedValidShadowCellQualifier =
67 com.google.common.primitives.Bytes.concat(validShadowCellQualifier, com.google.common.primitives.Bytes
68 .concat(validShadowCellQualifier, validShadowCellQualifier));
69 final byte[] value = Bytes.toBytes("test-value");
70
71
72
73
74
75 KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
76 assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
77
78
79
80 kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
81 assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
82
83
84
85 kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
86 assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
87
88
89
90 kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
91 assertTrue(CellUtils.isShadowCell(kv), "Should include a valid shadowCell identifier");
92
93
94
95 kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
96 assertFalse(CellUtils.isShadowCell(kv), "Should not include a valid shadowCell identifier");
97
98 }
99
100 @Test(timeOut = 10_000)
101 public void testCorrectMapingOfCellsToShadowCells() throws IOException {
102
103 final byte[] validShadowCellQualifier =
104 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
105
106 final byte[] qualifier2 = Bytes.toBytes("test-qual2");
107 final byte[] validShadowCellQualifier2 =
108 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier2, SHADOW_CELL_SUFFIX);
109
110 final byte[] qualifier3 = Bytes.toBytes("test-qual3");
111
112 Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
113 Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
114 Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
115 Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
116 Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
117
118 Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
119 Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
120
121 Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
122
123
124 List<Cell> badListWithDups = new ArrayList<>();
125 badListWithDups.add(cell1);
126 badListWithDups.add(dupCell1WithAnotherValue);
127
128
129 SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
130 assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
131 assertTrue(cellsToShadowCells.containsKey(cell1));
132 KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
133 KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
134 assertTrue(firstKey.equals(lastKey));
135 assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
136 cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()),
137 "Should be equal");
138
139
140 HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
141 cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
142 assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
143 assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
144 firstKey = (KeyValue) cellsToShadowCells.firstKey();
145 lastKey = (KeyValue) cellsToShadowCells.lastKey();
146 assertTrue(firstKey.equals(lastKey));
147 assertTrue(0 == Bytes.compareTo(firstKey.getValueArray(), firstKey.getValueOffset(),
148 firstKey.getValueLength(), dupCell1WithAnotherValue.getValueArray(),
149 dupCell1WithAnotherValue.getValueOffset(), dupCell1WithAnotherValue.getValueLength()),
150 "Should be equal");
151
152 List<Cell> cellListWithDups = new ArrayList<>();
153 cellListWithDups.add(cell1);
154 cellListWithDups.add(shadowCell1);
155 cellListWithDups.add(dupCell1);
156 cellListWithDups.add(delCell1);
157 cellListWithDups.add(cell2);
158 cellListWithDups.add(cell3);
159 cellListWithDups.add(shadowCell2);
160
161 cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
162 assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
163 assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
164 assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
165 assertFalse(cellsToShadowCells.containsKey(delCell1));
166
167
168 assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
169 assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
170
171 }
172
173 @Test(timeOut = 10_000)
174 public void testShadowCellSuffixConcatenationToQualifier() {
175
176 Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
177 byte[] suffixedQualifier = CellUtils.addShadowCellSuffixPrefix(cell.getQualifierArray(),
178 cell.getQualifierOffset(),
179 cell.getQualifierLength());
180 byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
181 assertEquals(suffixedQualifier, expectedQualifier);
182
183 }
184
185 @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
186 public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
187
188
189 byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, shadowCellSuffixToTest);
190 Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
191 byte[] resultedQualifier = CellUtils.removeShadowCellSuffixPrefix(cell.getQualifierArray(),
192 cell.getQualifierOffset(),
193 cell.getQualifierLength());
194 byte[] expectedQualifier = qualifier;
195 assertEquals(resultedQualifier, expectedQualifier);
196
197
198 byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, Bytes.toBytes("BAD"));
199 Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
200 try {
201 CellUtils.removeShadowCellSuffixPrefix(badCell.getQualifierArray(),
202 badCell.getQualifierOffset(),
203 badCell.getQualifierLength());
204 fail();
205 } catch (IllegalArgumentException e) {
206
207 }
208 }
209
210 @Test(timeOut = 10_000)
211 public void testMatchingQualifiers() {
212 Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
213 assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
214 assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
215 }
216
217 @Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
218 public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
219
220 byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, shadowCellSuffixToTest);
221 int originalQualifierLength =
222 CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
223 assertEquals(originalQualifierLength, qualifier.length);
224
225
226 originalQualifierLength =
227 CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
228 assertEquals(originalQualifierLength, qualifier.length);
229 }
230
231
232 @Test(timeOut = 10_000)
233 public void testmapCellsToShadowCellsCellOrder() {
234
235 final byte[] validShadowCellQualifier =
236 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier, SHADOW_CELL_SUFFIX);
237
238 final byte[] qualifier2 = Bytes.toBytes("test-qual2");
239 final byte[] validShadowCellQualifier2 =
240 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier2, SHADOW_CELL_SUFFIX);
241
242 final byte[] qualifier3 = Bytes.toBytes("test-qual3");
243 final byte[] validShadowCellQualifier3 =
244 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier3, SHADOW_CELL_SUFFIX);
245
246 final byte[] qualifier4 = Bytes.toBytes("test-qual4");
247 final byte[] qualifier5 = Bytes.toBytes("test-qual5");
248 final byte[] validShadowCellQualifier5 =
249 com.google.common.primitives.Bytes.concat(SHADOW_CELL_PREFIX, qualifier5, SHADOW_CELL_SUFFIX);
250
251
252 Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
253 Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
254
255 Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
256 Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
257
258 Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
259 Cell shadowCell3 = new KeyValue(row, family, validShadowCellQualifier3, 1, Bytes.toBytes("sc-value2"));
260
261 Cell cell4 = new KeyValue(row, family, qualifier4, 1, Bytes.toBytes("value4"));
262
263 Cell shadowCell5 = new KeyValue(row, family, validShadowCellQualifier5, 1, Bytes.toBytes("sc-value2"));
264
265 List<Cell> scanList = new ArrayList<>();
266 scanList.add(shadowCell5);
267 scanList.add(cell3);
268 scanList.add(cell1);
269 scanList.add(shadowCell1);
270 scanList.add(shadowCell2);
271 scanList.add(cell4);
272 scanList.add(cell2);
273 scanList.add(shadowCell3);
274 scanList.add(shadowCell5);
275
276 SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(scanList);
277 assertEquals(cellsToShadowCells.get(cell1).get(), shadowCell1);
278 assertEquals(cellsToShadowCells.get(cell2).get(), shadowCell2);
279 assertEquals(cellsToShadowCells.get(cell3).get(), shadowCell3);
280 assertFalse(cellsToShadowCells.get(cell4).isPresent());
281 }
282
283 }