engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wave10a: 462/209 hudi/iceberg/beam-clean/samza-clean

Browse source
This commit is contained in:
russell@unturf.com 2026-03-27 17:10:09 -04:00
parent ab10b2f555
commit f7fa333977
13 changed files with 979 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

186
defects/iceberg/unit/IcebergSchemaUpdateDeletesAlgorithm.java Normal file
View file

@ -0,0 +1,186 @@
package unit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* iceberg-0001: SchemaUpdate.ApplyChanges List<Integer> deletes O(F×D) → HashSet O(F)
*
* Simulates the ApplyChanges.field() visitor pattern: called once per schema field,
* each call checks if the field is in the deletes collection.
*
* Compile: javac -d . IcebergSchemaUpdateDeletesAlgorithm.java
* Run: java -ea unit.IcebergSchemaUpdateDeletesAlgorithm
*/
public class IcebergSchemaUpdateDeletesAlgorithm {
static long compareCount = 0;
// Defective: List<Integer> deletes — O(D) per field visit
static class ApplyChanges_slow {
private final List<Integer> deletes;
ApplyChanges_slow(List<Integer> deletes) {
this.deletes = deletes;
}
// Called once per field in schema traversal
boolean isDeleted(int fieldId) {
for (int d : deletes) {
compareCount++;
if (d == fieldId) return true;
}
return false;
}
int visitSchema(int[] fieldIds) {
int deletedCount = 0;
for (int fid : fieldIds) {
if (isDeleted(fid)) deletedCount++;
}
return deletedCount;
}
}
// Fixed: Set<Integer> deletes — O(1) per field visit
static class ApplyChanges_fast {
private final Set<Integer> deletes;
ApplyChanges_fast(Set<Integer> deletes) {
this.deletes = deletes;
}
boolean isDeleted(int fieldId) {
compareCount++; // one hash lookup
return deletes.contains(fieldId);
}
int visitSchema(int[] fieldIds) {
int deletedCount = 0;
for (int fid : fieldIds) {
if (isDeleted(fid)) deletedCount++;
}
return deletedCount;
}
}
static void assertEq(String label, Object expected, Object actual) {
if (!expected.equals(actual)) {
throw new AssertionError(label + ": expected " + expected + " but got " + actual);
}
}
public static void main(String[] args) {
int pass = 0;
int fail = 0;
System.out.println("iceberg-0001: SchemaUpdate.ApplyChanges deletes List → HashSet");
// Test 1: correctness — same deleted count for slow and fast
try {
int fieldCount = 50;
int deleteCount = 10;
int[] fields = new int[fieldCount];
for (int i = 0; i < fieldCount; i++) fields[i] = i;
List<Integer> deleteList = new ArrayList<>();
Set<Integer> deleteSet = new HashSet<>();
for (int i = 0; i < deleteCount; i++) {
deleteList.add(i * 5);
deleteSet.add(i * 5);
}
compareCount = 0;
int slowDeleted = new ApplyChanges_slow(deleteList).visitSchema(fields);
compareCount = 0;
int fastDeleted = new ApplyChanges_fast(deleteSet).visitSchema(fields);
assertEq("deleted-count", slowDeleted, fastDeleted);
System.out.println(" PASS correctness-F=50-D=10 deleted=" + slowDeleted);
pass++;
} catch (AssertionError e) { System.out.println(" FAIL correctness: " + e.getMessage()); fail++; }
// Test 2: no deletes — correctness
try {
int[] fields = new int[]{1, 2, 3, 4, 5};
int slow = new ApplyChanges_slow(new ArrayList<>()).visitSchema(fields);
int fast = new ApplyChanges_fast(new HashSet<>()).visitSchema(fields);
assertEq("no-deletes", slow, fast);
assertEq("no-deletes-zero", 0, slow);
System.out.println(" PASS no-deletes");
pass++;
} catch (AssertionError e) { System.out.println(" FAIL no-deletes: " + e.getMessage()); fail++; }
// Test 3: all fields deleted — correctness
try {
int[] fields = new int[]{10, 20, 30};
List<Integer> dl = new ArrayList<>(); dl.add(10); dl.add(20); dl.add(30);
Set<Integer> ds = new HashSet<>(dl);
int slow = new ApplyChanges_slow(dl).visitSchema(fields);
int fast = new ApplyChanges_fast(ds).visitSchema(fields);
assertEq("all-deleted", slow, fast);
assertEq("all-deleted-count", 3, slow);
System.out.println(" PASS all-deleted");
pass++;
} catch (AssertionError e) { System.out.println(" FAIL all-deleted: " + e.getMessage()); fail++; }
// Test 4: operation count at F=1000, D=100
{
int F = 1000, D = 100;
int[] fields = new int[F];
for (int i = 0; i < F; i++) fields[i] = i;
List<Integer> dl = new ArrayList<>();
Set<Integer> ds = new HashSet<>();
for (int i = 0; i < D; i++) { dl.add(i); ds.add(i); }
compareCount = 0;
new ApplyChanges_slow(dl).visitSchema(fields);
long slowOps = compareCount;
compareCount = 0;
new ApplyChanges_fast(ds).visitSchema(fields);
long fastOps = compareCount;
System.out.printf(" F=1000 D=100: slow=%8d fast=%6d ratio=%4dx%n",
slowOps, fastOps, slowOps / fastOps);
if (slowOps > fastOps * 10) {
System.out.println(" PASS slow > 10x fast at F=1000 D=100"); pass++;
} else {
System.out.println(" FAIL expected slow > 10x fast"); fail++;
}
}
// Test 5: F=500, D=50 — wide schema typical for Iceberg analytics
{
int F = 500, D = 50;
int[] fields = new int[F];
for (int i = 0; i < F; i++) fields[i] = i;
List<Integer> dl = new ArrayList<>();
Set<Integer> ds = new HashSet<>();
for (int i = 0; i < D; i++) { dl.add(i * 10); ds.add(i * 10); }
compareCount = 0;
int slowResult = new ApplyChanges_slow(dl).visitSchema(fields);
long slowOps = compareCount;
compareCount = 0;
int fastResult = new ApplyChanges_fast(ds).visitSchema(fields);
long fastOps = compareCount;
assertEq("wide-schema-result", slowResult, fastResult);
System.out.printf(" F=500 D=50: slow=%8d fast=%6d ratio=%4dx%n",
slowOps, fastOps, slowOps / fastOps);
if (slowOps > fastOps * 5) {
System.out.println(" PASS slow > 5x fast at F=500 D=50"); pass++;
} else {
System.out.println(" FAIL expected slow > 5x fast"); fail++;
}
}
System.out.println(pass + "/" + (pass + fail) + " PASS");
if (fail > 0) throw new RuntimeException(fail + " tests failed");
}
}