Skip to content

Commit ffaea42

Browse files
DEADripERDEADripER
authored
IGNITE-27716 Add group name to DumpReader logs (#12840)
1 parent d2acf96 commit ffaea42

2 files changed

Lines changed: 208 additions & 6 deletions

File tree

‎modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -129,19 +129,20 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) {
129129

130130
for (Map.Entry<Integer, List<String>> e : grpsCfgs.grpToNodes.entrySet()) {
131131
int grp = e.getKey();
132+
String grpName = grpsCfgs.grpIdToName.get(grp);
132133

133134
for (String node : e.getValue()) {
134135
for (int part : dump.partitions(node, grp)) {
135136
if (grps != null && !grps.get(grp).add(part)) {
136-
log.info("Skip copy partition [node=" + node + ", grp=" + grp + ", part=" + part + ']');
137+
log.info("Skip copy partition [node=" + node + ", grp=" + grpName + ", part=" + part + ']');
137138

138139
continue;
139140
}
140141

141142
Runnable consumePart = () -> {
142143
if (skip.get()) {
143144
if (log.isDebugEnabled()) {
144-
log.debug("Skip partition due to previous error [node=" + node + ", grp=" + grp +
145+
log.debug("Skip partition due to previous error [node=" + node + ", grp=" + grpName +
145146
", part=" + part + ']');
146147
}
147148

@@ -150,7 +151,7 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) {
150151

151152
try (DumpedPartitionIterator iter = dump.iterator(node, grp, part, grpsCfgs.cacheIds)) {
152153
if (log.isDebugEnabled()) {
153-
log.debug("Consuming partition [node=" + node + ", grp=" + grp +
154+
log.debug("Consuming partition [node=" + node + ", grp=" + grpName +
154155
", part=" + part + ']');
155156
}
156157

@@ -159,7 +160,7 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) {
159160
catch (Exception ex) {
160161
skip.set(cfg.failFast());
161162

162-
log.error("Error consuming partition [node=" + node + ", grp=" + grp +
163+
log.error("Error consuming partition [node=" + node + ", grp=" + grpName +
163164
", part=" + part + ']', ex);
164165

165166
throw new IgniteException(ex);
@@ -365,6 +366,7 @@ private static GridKernalContext standaloneKernalContext(SnapshotFileTree sft, I
365366
private GroupsConfigs groupsConfigs(Dump dump) {
366367
Map<Integer, List<String>> grpsToNodes = new HashMap<>();
367368
List<StoredCacheData> ccfgs = new ArrayList<>();
369+
Map<Integer, String> grpIdToName = new HashMap<>();
368370

369371
Set<Integer> grpIds = cfg.groupNames() != null
370372
? Arrays.stream(cfg.groupNames()).map(CU::cacheId).collect(Collectors.toSet())
@@ -392,11 +394,13 @@ private GroupsConfigs groupsConfigs(Dump dump) {
392394
}
393395

394396
grpsToNodes.get(grp).add(meta.folderName());
397+
398+
grpIdToName.put(grp, CU.cacheOrGroupName(grpCaches.get(0).configuration()));
395399
}
396400
}
397401

398402
// Optimize - skip whole cache if only one in group!
399-
return new GroupsConfigs(grpsToNodes, ccfgs, cacheIds);
403+
return new GroupsConfigs(grpsToNodes, ccfgs, cacheIds, grpIdToName);
400404
}
401405

402406
/** */
@@ -410,11 +414,20 @@ private static class GroupsConfigs {
410414
/** Cache ids. */
411415
public final Set<Integer> cacheIds;
412416

417+
/** Mapping from group id to group name. */
418+
public final Map<Integer, String> grpIdToName;
419+
413420
/** */
414-
public GroupsConfigs(Map<Integer, List<String>> grpToNodes, Collection<StoredCacheData> cacheCfgs, Set<Integer> cacheIds) {
421+
public GroupsConfigs(
422+
Map<Integer, List<String>> grpToNodes,
423+
Collection<StoredCacheData> cacheCfgs,
424+
Set<Integer> cacheIds,
425+
Map<Integer, String> grpIdToName
426+
) {
415427
this.grpToNodes = grpToNodes;
416428
this.cacheCfgs = cacheCfgs;
417429
this.cacheIds = cacheIds;
430+
this.grpIdToName = grpIdToName;
418431
}
419432
}
420433
}

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/IgniteCacheDumpSelf2Test.java‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@
121121
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_TRANSFER_RATE_DMS_KEY;
122122
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.CACHE_0;
123123
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.DMP_NAME;
124+
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.GRP;
124125
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.KEYS_CNT;
125126
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.USER_FACTORY;
126127
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.AbstractCacheDumpTest.dump;
@@ -1296,6 +1297,194 @@ public void testConfigOnlySnapshotThrows() throws Exception {
12961297
}
12971298
}
12981299

1300+
/** */
1301+
@Test
1302+
public void testDumpReaderDebugLogsGroupName() throws Exception {
1303+
checkDumpReaderDebugLogsGroupName(GRP, GRP, new String[]{GRP});
1304+
}
1305+
1306+
/** */
1307+
@Test
1308+
public void testDumpReaderDebugLogsNullableGroupName() throws Exception {
1309+
checkDumpReaderDebugLogsGroupName(GRP, GRP, null);
1310+
}
1311+
1312+
/** */
1313+
@Test
1314+
public void testDumpReaderDebugLogsNoGroupName() throws Exception {
1315+
checkDumpReaderDebugLogsGroupName(null, CACHE_0, null);
1316+
}
1317+
1318+
/** */
1319+
@Test
1320+
public void testDumpReaderSkipCopiesLogsGroupName() throws Exception {
1321+
int parts = 4;
1322+
String dumpName0 = "dump0";
1323+
String dumpName1 = "dump1";
1324+
1325+
File snapshotPath0 = Files.createTempDirectory("snapshots0").toFile();
1326+
File snapshotPath1 = Files.createTempDirectory("snapshots1").toFile();
1327+
File combinedDumpDir = Files.createTempDirectory("combined_dump").toFile();
1328+
1329+
ListeningTestLogger testLog = new ListeningTestLogger(log);
1330+
1331+
LogListener skipLsnr = LogListener.matches("Skip copy partition")
1332+
.times(parts)
1333+
.andMatches("grp=" + GRP)
1334+
.build();
1335+
1336+
testLog.registerListener(skipLsnr);
1337+
1338+
try {
1339+
IgniteEx node0 = startGrid(getConfiguration("node0")
1340+
.setConsistentId("node0")
1341+
.setSnapshotPath(snapshotPath0.getAbsolutePath())
1342+
.setGridLogger(testLog));
1343+
1344+
IgniteEx node1 = startGrid(getConfiguration("node1")
1345+
.setConsistentId("node1")
1346+
.setSnapshotPath(snapshotPath1.getAbsolutePath())
1347+
.setGridLogger(testLog));
1348+
1349+
node0.cluster().state(ClusterState.ACTIVE);
1350+
1351+
CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<Integer, Integer>()
1352+
.setName(DEFAULT_CACHE_NAME)
1353+
.setGroupName(GRP)
1354+
.setBackups(1)
1355+
.setAffinity(new RendezvousAffinityFunction().setPartitions(parts));
1356+
1357+
IgniteCache<Integer, Integer> cache = node0.createCache(ccfg);
1358+
1359+
IntStream.range(0, KEYS_CNT).forEach(i -> cache.put(i, i));
1360+
1361+
node0.snapshot().createDump(dumpName0, null).get(getTestTimeout());
1362+
1363+
U.sleep(100);
1364+
1365+
node1.snapshot().createDump(dumpName1, null).get(getTestTimeout());
1366+
1367+
File dumpDir0 = new File(snapshotPath0, dumpName0);
1368+
File dumpDir1 = new File(snapshotPath1, dumpName1);
1369+
1370+
U.copy(dumpDir0, combinedDumpDir, true);
1371+
U.copy(dumpDir1, combinedDumpDir, true);
1372+
1373+
DumpConsumer dummyConsumer = new DumpConsumer() {
1374+
@Override public void start() {
1375+
// No-op.
1376+
}
1377+
1378+
@Override public void onMappings(Iterator<TypeMapping> mappings) {
1379+
// No-op.
1380+
}
1381+
1382+
@Override public void onTypes(Iterator<BinaryType> types) {
1383+
// No-op.
1384+
}
1385+
1386+
@Override public void onCacheConfigs(Iterator<StoredCacheData> caches) {
1387+
// No-op.
1388+
}
1389+
1390+
@Override public void onPartition(int grpId, int partId, Iterator<DumpEntry> data) {
1391+
// No-op.
1392+
}
1393+
1394+
@Override public void stop() {
1395+
// No-op.
1396+
}
1397+
};
1398+
1399+
new DumpReader(
1400+
new DumpReaderConfiguration(
1401+
null,
1402+
combinedDumpDir.getAbsolutePath(),
1403+
null,
1404+
dummyConsumer,
1405+
DFLT_THREAD_CNT,
1406+
DFLT_TIMEOUT,
1407+
false,
1408+
true,
1409+
true,
1410+
new String[]{GRP},
1411+
null,
1412+
true,
1413+
null
1414+
),
1415+
testLog
1416+
).run();
1417+
1418+
assertTrue(skipLsnr.check());
1419+
}
1420+
finally {
1421+
U.delete(snapshotPath0);
1422+
U.delete(snapshotPath1);
1423+
U.delete(combinedDumpDir);
1424+
}
1425+
}
1426+
1427+
/** */
1428+
private void checkDumpReaderDebugLogsGroupName(
1429+
@Nullable String cacheGroupName,
1430+
String expectedLogGroup,
1431+
@Nullable String[] grpNames
1432+
) throws Exception {
1433+
String id = "test";
1434+
setLoggerDebugLevel();
1435+
ListeningTestLogger testLog = new ListeningTestLogger(log);
1436+
1437+
LogListener errLsnr = LogListener.matches("Error consuming partition").andMatches("grp=" + expectedLogGroup).build();
1438+
LogListener cnsmLsnr = LogListener.matches("Consuming partition").andMatches("grp=" + expectedLogGroup).build();
1439+
1440+
testLog.registerListener(errLsnr);
1441+
testLog.registerListener(cnsmLsnr);
1442+
1443+
IgniteEx ign = startGrid(getConfiguration(id).setConsistentId(id).setGridLogger(testLog));
1444+
ign.cluster().state(ClusterState.ACTIVE);
1445+
1446+
CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<Integer, Integer>()
1447+
.setName(CACHE_0)
1448+
.setBackups(1)
1449+
.setAffinity(new RendezvousAffinityFunction().setPartitions(3));
1450+
1451+
if (cacheGroupName != null)
1452+
ccfg.setGroupName(cacheGroupName);
1453+
1454+
IgniteCache<Integer, Integer> cache = ign.createCache(ccfg);
1455+
IntStream.range(0, KEYS_CNT).forEach(i -> cache.put(i, i));
1456+
1457+
ign.snapshot().createDump(DMP_NAME, null).get(getTestTimeout());
1458+
1459+
TestDumpConsumer cnsmr = new TestDumpConsumer() {
1460+
@Override public void onPartition(int grp, int part, Iterator<DumpEntry> data) {
1461+
throw new RuntimeException("trigger error log");
1462+
}
1463+
};
1464+
1465+
assertThrows(null, () -> new DumpReader(
1466+
new DumpReaderConfiguration(
1467+
DMP_NAME,
1468+
null,
1469+
ign.configuration(),
1470+
cnsmr,
1471+
DFLT_THREAD_CNT,
1472+
DFLT_TIMEOUT,
1473+
true,
1474+
true,
1475+
false,
1476+
grpNames,
1477+
null,
1478+
false,
1479+
null
1480+
),
1481+
testLog
1482+
).run(), RuntimeException.class, "trigger error log");
1483+
1484+
assertTrue("Log with group name not found", errLsnr.check());
1485+
assertTrue("Consuming with group name not found", cnsmLsnr.check());
1486+
}
1487+
12991488
/** */
13001489
public class TestCacheConflictResolutionManager<K, V> extends GridCacheManagerAdapter<K, V>
13011490
implements CacheConflictResolutionManager<K, V> {

0 commit comments

Comments
 (0)