diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3b161..779f274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Synthetic data generator +**Version 2.7** + +- Fixed issue in date of diagnosis rule that would produce DX dates outside of the min/max DX dates provided in the options. + **Version 2.6** - Fixed issue in DOLC that would produce tumors with vital status 0 (dead) and DX date later than DOLC. diff --git a/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java b/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java index 2c40f14..1099a47 100644 --- a/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java +++ b/src/main/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRule.java @@ -1,9 +1,7 @@ package com.imsweb.datagenerator.naaccr.rule.tumor; import java.time.LocalDate; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import com.imsweb.datagenerator.naaccr.NaaccrDataGeneratorOptions; import com.imsweb.datagenerator.naaccr.NaaccrDataGeneratorTumorRule; @@ -29,51 +27,56 @@ public DateOfDiagnosisRule() { @Override public void execute(Tumor tumor, Patient patient, NaaccrDataGeneratorOptions options, Map context) { - // latest possible date set only by options if defined - Set maxDxDates = new HashSet<>(); + // the date range requested in the options is a hard requirement; the other constraints (age group of the site, year of birth, dx date of the + // previous tumor) are only applied when they don't push the generated date outside of that range LocalDate maxDate = options == null ? LocalDate.now() : options.getMaxDxDate(); - maxDxDates.add(maxDate); + LocalDate minDate = options == null ? LocalDate.now().minusYears(10) : options.getMinDxDate(); + // the min dx date defaults to ten years ago when it's not provided in the options; that default can end up after a requested max dx date + if (minDate.isAfter(maxDate)) + minDate = maxDate.minusYears(10); - Set minDxDates = new HashSet<>(); - // never go before min date defined in options, or current date minus ten years if options not defined - minDxDates.add(options == null ? LocalDate.now().minusYears(10) : options.getMinDxDate()); // never go before the year of birth if (hasValue(patient, "dateOfBirthYear")) - minDxDates.add(LocalDate.of(Integer.parseInt(patient.getItemValue("dateOfBirthYear")) + 1, 1, 1)); + minDate = tightenMinDate(minDate, maxDate, LocalDate.of(Integer.parseInt(patient.getItemValue("dateOfBirthYear")) + 1, 1, 1)); + // never go before dx date of patient's most recent tumor (if this isn't the first one) if (!patient.getTumors().isEmpty()) { Tumor lastTumor = patient.getTumor(patient.getTumors().size() - 1); - minDxDates.add(LocalDate.of( + minDate = tightenMinDate(minDate, maxDate, LocalDate.of( Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisYear")), Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisMonth")), Integer.parseInt(lastTumor.getItemValue("dateOfDiagnosisDay")))); } - if (context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX) != null) { - int birthYear = Integer.parseInt(patient.getItemValue("dateOfBirthYear")); - int birthMonth = Integer.parseInt(patient.getItemValue("dateOfBirthMonth")); - int birthDay = Integer.parseInt(patient.getItemValue("dateOfBirthDay")); - LocalDate dateOfBirth = LocalDate.of(birthYear, birthMonth, birthDay); - - // PROBLEM: This brakes 3 previous rules: - // 1. Minimum date must be within 10 years of today. - // 2. Options specify a minimum DX date. - // 3. This tumor must be diagnosed after the previous ones for this patient. - // From Fabian: Only #2 is required. Try to get all tumors to use this minimum. If that can't be done, at least one tumor must meet it. - minDxDates.clear(); - maxDxDates.clear(); - - int currentTumorIndex = (int)context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX); - @SuppressWarnings("unchecked") - Map ageGroupMap = (Map)context.get(CONTEXT_FLAG_AGE_GROUP_MAP); - minDxDates.add(dateOfBirth.plusYears((ageGroupMap.get(currentTumorIndex) * 10))); - maxDxDates.add(maxDate); + // never diagnose the tumor before the patient reaches the age group that was picked for its site + @SuppressWarnings("unchecked") + Map ageGroupMap = (Map)context.get(CONTEXT_FLAG_AGE_GROUP_MAP); + Integer currentTumorIndex = (Integer)context.get(CONTEXT_FLAG_CURRENT_TUMOR_INDEX); + if (ageGroupMap != null && currentTumorIndex != null && hasValue(patient, "dateOfBirthYear", "dateOfBirthMonth", "dateOfBirthDay")) { + Integer ageGroup = ageGroupMap.get(currentTumorIndex); + // the age group is -1 for a site that has no age distribution, in which case it tells us nothing about the dx date + if (ageGroup != null && ageGroup > 0) { + LocalDate dateOfBirth = LocalDate.of( + Integer.parseInt(patient.getItemValue("dateOfBirthYear")), + Integer.parseInt(patient.getItemValue("dateOfBirthMonth")), + Integer.parseInt(patient.getItemValue("dateOfBirthDay"))); + minDate = tightenMinDate(minDate, maxDate, dateOfBirth.plusYears(ageGroup * 10L)); + } } - LocalDate randomDate = RandomUtils.getRandomDateBetween(minDxDates, maxDxDates); + LocalDate randomDate = RandomUtils.getRandomDateBetween(minDate, maxDate); setValue(tumor, "dateOfDiagnosisYear", Integer.toString(randomDate.getYear())); setValue(tumor, "dateOfDiagnosisMonth", Integer.toString(randomDate.getMonthValue())); setValue(tumor, "dateOfDiagnosisDay", Integer.toString(randomDate.getDayOfMonth())); } + + /** + * Returns the candidate date if it is a tighter minimum than the current one and still leaves a valid range, the current minimum otherwise. + */ + private static LocalDate tightenMinDate(LocalDate currentMinDate, LocalDate maxDate, LocalDate candidate) { + if (candidate.isAfter(currentMinDate) && !candidate.isAfter(maxDate)) + return candidate; + return currentMinDate; + } } diff --git a/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java b/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java index 66fd4c2..f17892b 100644 --- a/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java +++ b/src/test/java/com/imsweb/datagenerator/naaccr/NaaccrFixedColumnsDataGeneratorTest.java @@ -144,22 +144,20 @@ public void testGeneratePatient() throws IOException { patient = generator.generatePatient(1, options); Assert.assertEquals("TEST", patient.getFirst().get("nameLast")); - // Test context - int numTumors = 2; + // Test context; the requested dx date range applies to every tumor, not just to one of them + int numTumors = 3; options = new NaaccrDataGeneratorOptions(); options.setMinDxYear(2000); options.setMaxDxYear(2005); - patient = generator.generatePatient(numTumors, options); - - LocalDate dateOfDx1 = LocalDate.of(Integer.parseInt(patient.get(0).get("dateOfDiagnosisYear")), Integer.parseInt(patient.get(0).get("dateOfDiagnosisMonth")), - Integer.parseInt(patient.get(0).get("dateOfDiagnosisDay"))); - LocalDate dateOfDx2 = LocalDate.of(Integer.parseInt(patient.get(1).get("dateOfDiagnosisYear")), Integer.parseInt(patient.get(1).get("dateOfDiagnosisMonth")), - Integer.parseInt(patient.get(1).get("dateOfDiagnosisDay"))); - - boolean dateInRange1 = dateOfDx1.isAfter(options.getMinDxDate().minusDays(1)) && dateOfDx1.isBefore(options.getMaxDxDate().plusDays(1)); - boolean dateInRange2 = dateOfDx2.isAfter(options.getMinDxDate().minusDays(1)) && dateOfDx2.isBefore(options.getMaxDxDate().plusDays(1)); - - Assert.assertTrue("Diagnosis Date outside options Minimum and Maximum.", dateInRange1 || dateInRange2); + for (int i = 0; i < 250; i++) { + patient = generator.generatePatient(numTumors, options); + for (Map rec : patient) { + LocalDate dateOfDx = LocalDate.of(Integer.parseInt(rec.get("dateOfDiagnosisYear")), Integer.parseInt(rec.get("dateOfDiagnosisMonth")), + Integer.parseInt(rec.get("dateOfDiagnosisDay"))); + Assert.assertFalse("Diagnosis Date before options Minimum: " + dateOfDx, dateOfDx.isBefore(options.getMinDxDate())); + Assert.assertFalse("Diagnosis Date after options Maximum: " + dateOfDx, dateOfDx.isAfter(options.getMaxDxDate())); + } + } // another test with an incidence generator generator = new NaaccrFixedColumnsDataGenerator(LayoutFactory.LAYOUT_ID_NAACCR_18_INCIDENCE); diff --git a/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java b/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java index 498697b..910c3a3 100644 --- a/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java +++ b/src/test/java/com/imsweb/datagenerator/naaccr/rule/tumor/DateOfDiagnosisRuleTest.java @@ -101,7 +101,85 @@ public void testExecute() { LocalDate startDate = dateOfBirth.plusYears(5 * 10); LocalDate endDate = options.getMaxDxDate(); - Assert.assertTrue(dateOfDx.toString(), dateOfDx.isAfter(startDate) && dateOfDx.isBefore(endDate)); + Assert.assertTrue(dateOfDx.toString(), !dateOfDx.isBefore(startDate) && !dateOfDx.isAfter(endDate)); } + + @Test + public void testExecuteRespectsRequestedDxDateRange() { + + NaaccrDataGeneratorOptions options = new NaaccrDataGeneratorOptions(); + options.setMinDxYear(2015); + options.setMaxDxYear(2020); + + // the birth date of a patient is based on the oldest age group of all its tumors (see BirthRule), so a tumor assigned to a + // younger age group used to be allowed to go decades before the requested min dx date + Map ageGroupMap = new HashMap<>(); + ageGroupMap.put(0, 7); + ageGroupMap.put(1, 2); + + Map context = new HashMap<>(); + context.put(CONTEXT_FLAG_AGE_GROUP_MAP, ageGroupMap); + context.put(CONTEXT_FLAG_MAX_AGE_GROUP, 7); + + for (int i = 0; i < 100; i++) { + Patient patient = new Patient(); + patient.addItem(new Item("dateOfBirthYear", "1945")); + patient.addItem(new Item("dateOfBirthMonth", "6")); + patient.addItem(new Item("dateOfBirthDay", "15")); + + LocalDate previousDxDate = null; + for (int tumorIdx = 0; tumorIdx < ageGroupMap.size(); tumorIdx++) { + context.put(CONTEXT_FLAG_CURRENT_TUMOR_INDEX, tumorIdx); + + Tumor tumor = new Tumor(); + _rule.execute(tumor, patient, options, context); + patient.addTumor(tumor); + + LocalDate dxDate = LocalDate.of( + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisYear")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisMonth")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisDay"))); + + Assert.assertFalse("Dx date before requested min dx date: " + dxDate, dxDate.isBefore(options.getMinDxDate())); + Assert.assertFalse("Dx date after requested max dx date: " + dxDate, dxDate.isAfter(options.getMaxDxDate())); + if (previousDxDate != null) + Assert.assertFalse("Dx date before previous tumor dx date: " + dxDate, dxDate.isBefore(previousDxDate)); + previousDxDate = dxDate; + } + } + } + + @Test + public void testExecuteWhenMinDxDateCannotBeReached() { + + // the min dx date defaults to ten years ago; combined with a max dx year in the past, that default range is inverted + NaaccrDataGeneratorOptions options = new NaaccrDataGeneratorOptions(); + options.setMaxDxYear(2005); + + Map ageGroupMap = new HashMap<>(); + ageGroupMap.put(0, 5); + + Map context = new HashMap<>(); + context.put(CONTEXT_FLAG_AGE_GROUP_MAP, ageGroupMap); + context.put(CONTEXT_FLAG_CURRENT_TUMOR_INDEX, 0); + context.put(CONTEXT_FLAG_MAX_AGE_GROUP, 5); + + Patient patient = new Patient(); + patient.addItem(new Item("dateOfBirthYear", "1940")); + patient.addItem(new Item("dateOfBirthMonth", "7")); + patient.addItem(new Item("dateOfBirthDay", "1")); + + Tumor tumor = new Tumor(); + _rule.execute(tumor, patient, options, context); + + LocalDate dxDate = LocalDate.of( + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisYear")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisMonth")), + Integer.parseInt(tumor.getItemValue("dateOfDiagnosisDay"))); + + // the max dx date is the hard requirement, the dx date must fall in the ten years before it + Assert.assertFalse(dxDate.toString(), dxDate.isAfter(options.getMaxDxDate())); + Assert.assertFalse(dxDate.toString(), dxDate.isBefore(options.getMaxDxDate().minusYears(10))); + } }