From 853024acab8176d32edb0b6d6ff115a1c80fbb23 Mon Sep 17 00:00:00 2001 From: Brad <1676751+Sinkleberg@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:05:33 -0500 Subject: [PATCH] Support percentage-weighted choices in custom YAML providers --- docs/documentation/custom-providers.md | 25 +++- .../net/datafaker/service/FakeValues.java | 23 +++ .../datafaker/service/FakeValuesService.java | 6 + .../datafaker/service/WeightedYamlValues.java | 91 ++++++++++++ .../service/WeightedYamlProviderTest.java | 103 ++++++++++++++ .../service/WeightedYamlValuesTest.java | 131 ++++++++++++++++++ src/test/resources/weighted-custom.yml | 26 ++++ 7 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 src/main/java/net/datafaker/service/WeightedYamlValues.java create mode 100644 src/test/java/net/datafaker/service/WeightedYamlProviderTest.java create mode 100644 src/test/java/net/datafaker/service/WeightedYamlValuesTest.java create mode 100644 src/test/resources/weighted-custom.yml diff --git a/docs/documentation/custom-providers.md b/docs/documentation/custom-providers.md index 2ea26c8c2..bbfc078f2 100644 --- a/docs/documentation/custom-providers.md +++ b/docs/documentation/custom-providers.md @@ -63,7 +63,7 @@ This will print something like the following: ### Weighted random selection -> **Note**: Usage of weighted random selector is in the POC stage and is currently available only for custom hardcoded providers. +> **Note**: Usage of weighted random selector is in the POC stage. Custom YAML providers also support the percentage syntax described below. Since version 2.4.2 of Datafaker it's possible to use a random selector based on weights, add a `WeightedRandomSelector` and weighted values to your provider: @@ -154,6 +154,29 @@ en: bees: ['Bumblebee', 'Euglossine bee', 'Honeybee', 'Carpenter bee', 'Leaf-cutter bee', 'Mining bee'] ``` +### Weighted YAML choices + +Custom files loaded with `addPath` or `addUrl` can use a list of percentage-key mappings: + +```yaml +en: + faker: + insectsfromfile: + ants: + - "60%": "Driver ant" + - "40%": "#{other_ant}" + other_ant: ["Fire ant", "Harvester ant"] + domains: + - "90%": ".it" + - "10%": [".ee", ".lv", ".lt"] +``` + +Each entry must have exactly one quoted decimal percentage key and a string template or a non-empty list of string templates. Percentages must be finite, positive and total exactly 100 (decimal fractions such as `33.3%` are supported). Branch values must be unique, matching `WeightedRandomSelector`'s validation. Mixing weighted entries with ordinary entries, or nesting weighted choices inside a branch, is not supported. + +The weighted branch is selected first; a list branch then selects one entry uniformly. Templates resolve in the same way as ordinary custom-provider templates, including local and other provider references. Selection uses the current Faker's random service, so seeded Fakers remain reproducible. Percentages describe probabilities, not guaranteed counts in a finite sample. + +This syntax applies only to custom files. Ordinary lists, maps and scalar values, bundled locale data, and the Java weighted-selection API retain their existing behavior. An empty ordinary list still uses the existing fallback behavior. + ### Register provider Registering the provider would happen like this: diff --git a/src/main/java/net/datafaker/service/FakeValues.java b/src/main/java/net/datafaker/service/FakeValues.java index 0a297dcb1..f4cf98006 100644 --- a/src/main/java/net/datafaker/service/FakeValues.java +++ b/src/main/java/net/datafaker/service/FakeValues.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -46,12 +47,34 @@ private Map loadFromUrl() { try (var stream = url.openStream()) { Map result = readFromStream(stream); enrichMapWithJavaNames(result); + if (result != null) { + result.forEach((provider, value) -> prepareCustomValues(value, provider)); + } return result; } catch (IOException e) { throw new RuntimeException("Failed to read fake values from %s".formatted(url), e); } } + private void prepareCustomValues(Object node, String provider) { + if (node instanceof Map rawMap) { + @SuppressWarnings("unchecked") + Map map = (Map) rawMap; + map.replaceAll((key, value) -> { + if (value instanceof List list) { + WeightedYamlValues weighted = WeightedYamlValues.from(list, template -> { + List singleton = new ArrayList<>(List.of(template)); + prefixUnqualifiedExpressions(singleton, provider); + return (String) singleton.get(0); + }); + if (weighted != null) return weighted; + } + prepareCustomValues(value, provider); + return value; + }); + } + } + private Map loadValues() { Map result = loadFromUrl(); if (result != null) return result; diff --git a/src/main/java/net/datafaker/service/FakeValuesService.java b/src/main/java/net/datafaker/service/FakeValuesService.java index a8396c994..d00eb5c3b 100644 --- a/src/main/java/net/datafaker/service/FakeValuesService.java +++ b/src/main/java/net/datafaker/service/FakeValuesService.java @@ -142,6 +142,9 @@ public void addUrl(Locale locale, URL url) { public Object fetch(String key, FakerContext context) { List valuesArray = null; final Object o = fetchObject(key, context); + if (o instanceof WeightedYamlValues weighted) { + return weighted.select(context.getRandomService()); + } if (o instanceof List list) { valuesArray = list; final int size = valuesArray.size(); @@ -196,6 +199,9 @@ public String safeFetch(String key, FakerContext context, @Nullable String defau Object o = fetchObject(key, context); String str; if (o == null) return defaultIfNull; + if (o instanceof WeightedYamlValues weighted) { + return weighted.select(context.getRandomService()); + } if (o instanceof List list) { final List values = (List) list; final int size = values.size(); diff --git a/src/main/java/net/datafaker/service/WeightedYamlValues.java b/src/main/java/net/datafaker/service/WeightedYamlValues.java new file mode 100644 index 000000000..322045fd4 --- /dev/null +++ b/src/main/java/net/datafaker/service/WeightedYamlValues.java @@ -0,0 +1,91 @@ +package net.datafaker.service; + +import org.jspecify.annotations.Nullable; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.UnaryOperator; + +/** A custom YAML choice, parsed once but sampled using the current Faker's random service. */ +final class WeightedYamlValues { + private final List> choices; + + private WeightedYamlValues(List> choices) { + this.choices = List.copyOf(choices); + } + + @Nullable + static WeightedYamlValues from(List values, UnaryOperator qualify) { + // Ordinary lists (including lists of maps) retain their existing meaning. + boolean weighted = values.stream().anyMatch(value -> value instanceof Map map + && map.keySet().stream().anyMatch(key -> key instanceof String s && s.endsWith("%"))); + if (!weighted) return null; + + List> choices = new ArrayList<>(values.size()); + var uniqueValues = new HashSet(); + BigDecimal total = BigDecimal.ZERO; + for (Object value : values) { + if (!(value instanceof Map map) || map.size() != 1) { + throw invalid("each entry must be a single percentage-to-value mapping"); + } + var entry = map.entrySet().iterator().next(); + if (!(entry.getKey() instanceof String key) || !key.matches("[0-9]+(?:\\.[0-9]+)?%")) { + throw invalid("percentage keys must be decimal strings such as '60%'"); + } + BigDecimal percentage = new BigDecimal(key.substring(0, key.length() - 1)); + double weight = percentage.doubleValue(); + if (percentage.signum() <= 0 || !Double.isFinite(weight) || weight <= 0) { + throw invalid("percentages must be finite and positive"); + } + total = total.add(percentage); + Object branch = entry.getValue(); + Object qualified; + if (branch instanceof String string) { + qualified = qualify.apply(string); + } else if (branch instanceof List list && !list.isEmpty()) { + List strings = new ArrayList<>(list.size()); + for (Object item : list) { + if (!(item instanceof String string)) { + throw invalid("list branches must contain only string templates"); + } + strings.add(qualify.apply(string)); + } + qualified = List.copyOf(strings); + } else { + throw invalid("each branch must be a string template or a non-empty list of string templates"); + } + if (!uniqueValues.add(qualified)) { + throw invalid("branch values must be unique, as required by WeightedRandomSelector"); + } + choices.add(Map.of("weight", weight, "value", qualified)); + } + if (total.compareTo(BigDecimal.valueOf(100)) != 0) { + throw invalid("percentages must total 100"); + } + return new WeightedYamlValues(choices); + } + + String select(RandomService random) { + // The Java provider API accepts Random, while RandomService also supports other + // RandomGenerators. Forward its draw without creating an independent random stream. + Random delegate = new Random(0) { + @Override + public double nextDouble() { + return random.nextDouble(); + } + }; + Object selected = new WeightedRandomSelector(delegate).select(choices); + if (selected instanceof List list) { + return (String) list.get(list.size() == 1 ? 0 : random.nextInt(list.size())); + } + return (String) selected; + } + + private static IllegalArgumentException invalid(String reason) { + return new IllegalArgumentException("Invalid weighted YAML choices: " + reason); + } +} diff --git a/src/test/java/net/datafaker/service/WeightedYamlProviderTest.java b/src/test/java/net/datafaker/service/WeightedYamlProviderTest.java new file mode 100644 index 000000000..919cfdaa4 --- /dev/null +++ b/src/test/java/net/datafaker/service/WeightedYamlProviderTest.java @@ -0,0 +1,103 @@ +package net.datafaker.service; + +import net.datafaker.providers.base.BaseFaker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class WeightedYamlProviderTest { + @TempDir + Path directory; + + @Test + void resolvesCustomWeightedYamlAndPreservesOrdinaryValues() { + BaseFaker faker = custom(new Random(1)); + faker.addUrl(Locale.ENGLISH, getClass().getResource("/weighted-custom.yml")); + for (int i = 0; i < 20; i++) { + assertThat(faker.resolve("weighted_custom.strings")).isIn("first", "second"); + assertThat(faker.resolve("weighted_custom.domains")).isIn(".it", ".ee", ".lv", ".lt"); + assertThat(faker.resolve("weighted_custom.local_template")).isEqualTo("chosen!"); + assertThat(faker.resolve("weighted_custom.reference")).isNotEmpty(); + assertThat(faker.resolve("weighted_custom.ordinary")).isIn("a", "b"); + assertThat(faker.resolve("weighted_custom.scalar")).isEqualTo("plain"); + assertThat(faker.resolve("weighted_custom.nested.value")).isEqualTo("nested"); + } + assertThat(faker.fakeValuesService().fetchString("weighted_custom.strings", faker.getContext())).isIn("first", "second"); + Object ordinary = faker.fakeValuesService().fetchObject("weighted_custom.ordinary", faker.getContext()); + assertThat(ordinary).isEqualTo(java.util.List.of("a", "b")); + assertThat(faker.fakeValuesService().safeFetch("weighted_custom.empty", faker.getContext(), "fallback")).isEqualTo("fallback"); + // Custom data is merged with the ordinary locale fallback, not substituted for it. + assertThat(faker.resolve("name.first_name")).isNotEmpty(); + } + + @Test + void sameSeedAndSharedYamlDoNotShareSamples() { + BaseFaker first = custom(new Random(123)); + BaseFaker second = custom(new Random(123)); + for (BaseFaker faker : java.util.List.of(first, second)) { + faker.addUrl(Locale.ENGLISH, getClass().getResource("/weighted-custom.yml")); + } + for (int i = 0; i < 100; i++) { + assertThat(first.resolve("weighted_custom.domains")).isEqualTo(second.resolve("weighted_custom.domains")); + } + } + + @Test + void customSourcesAndRandomServicesAreIsolated() throws IOException { + Path a = directory.resolve("a.yml"); + Path b = directory.resolve("b.yml"); + Files.writeString(a, "en:\n faker:\n weighted_custom:\n value:\n - '100%': 'source-a'\n"); + Files.writeString(b, "en:\n faker:\n weighted_custom:\n value:\n - '100%': 'source-b'\n"); + BaseFaker first = custom(new Random(1)); + BaseFaker second = custom(new Random(1)); + first.addPath(Locale.ENGLISH, a); + second.addPath(Locale.ENGLISH, b); + assertThat(first.resolve("weighted_custom.value")).isEqualTo("source-a"); + assertThat(second.resolve("weighted_custom.value")).isEqualTo("source-b"); + first.getContext().setRandomService(new RandomService(new Random(2))); + assertThat(first.resolve("weighted_custom.value")).isEqualTo("source-a"); + } + + @Test + void preservesIntegerKeysInOrdinaryNestedMaps() throws IOException { + Path file = directory.resolve("integer-keys.yml"); + Files.writeString(file, """ + en: + faker: + weighted_custom: + sibling: 'ordinary sibling' + original: + 1: 'one' + 2: ['two', 'deux'] + """); + BaseFaker faker = custom(new Random(1)); + faker.addPath(Locale.ENGLISH, file); + assertThat(faker.resolve("weighted_custom.sibling")).isEqualTo("ordinary sibling"); + Object original = faker.fakeValuesService().fetchObject("weighted_custom.original", faker.getContext()); + assertThat(original).isEqualTo(java.util.Map.of(1, "one", 2, java.util.List.of("two", "deux"))); + } + + @Test + void malformedCustomYamlFailsClearly() throws IOException { + Path bad = directory.resolve("bad.yml"); + Files.writeString(bad, "en:\n faker:\n weighted_custom:\n value:\n - '60%': 'a'\n - '30%': 'b'\n"); + BaseFaker faker = custom(new Random(1)); + assertThatThrownBy(() -> { + faker.addPath(Locale.ENGLISH, bad); + faker.resolve("weighted_custom.value"); + }) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("percentages must total 100"); + } + + private static BaseFaker custom(Random random) { + return new BaseFaker(Locale.ENGLISH, random); + } +} diff --git a/src/test/java/net/datafaker/service/WeightedYamlValuesTest.java b/src/test/java/net/datafaker/service/WeightedYamlValuesTest.java new file mode 100644 index 000000000..0e998e48a --- /dev/null +++ b/src/test/java/net/datafaker/service/WeightedYamlValuesTest.java @@ -0,0 +1,131 @@ +package net.datafaker.service; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.UnaryOperator; +import java.util.random.RandomGenerator; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class WeightedYamlValuesTest { + @Test + void ordinaryListsAreNotWeighted() { + for (List list : List.of(List.of(), List.of("a", "b"), List.of(Map.of("plain", "value")))) { + assertThat(WeightedYamlValues.from(list, UnaryOperator.identity())).isNull(); + } + } + + @Test + void followsExistingSelectorAtCumulativeBoundaries() { + var values = WeightedYamlValues.from(List.of(Map.of("60%", "first"), Map.of("40%", "second")), UnaryOperator.identity()); + // Preserve the existing selector's exact-boundary convention, including 60 itself. + for (double draw : new double[]{0, Math.nextDown(0.6), 0.6}) { + assertThat(values.select(fixed(draw, 0))).isEqualTo("first"); + } + for (double draw : new double[]{Math.nextUp(0.6), Math.nextDown(1.0)}) { + assertThat(values.select(fixed(draw, 0))).isEqualTo("second"); + } + } + + @Test + void selectsBranchThenUniformListEntry() { + var values = WeightedYamlValues.from(List.of(Map.of("90%", ".it"), Map.of("10%", List.of(".ee", ".lv", ".lt"))), UnaryOperator.identity()); + assertThat(values.select(fixed(0, 2))).isEqualTo(".it"); + for (int i = 0; i < 3; i++) { + assertThat(values.select(fixed(0.99, i))).isEqualTo(List.of(".ee", ".lv", ".lt").get(i)); + } + } + + @Test + void supportsAnyFakerRandomGeneratorWithoutIndependentRandomness() { + RandomGenerator generator = new RandomGenerator() { + @Override + public long nextLong() { + throw new AssertionError("Unexpected independent random draw"); + } + + @Override + public double nextDouble() { + return 0.99; + } + }; + var values = WeightedYamlValues.from(List.of(Map.of("60%", "first"), Map.of("40%", "second")), UnaryOperator.identity()); + assertThat(values.select(new RandomService(generator))).isEqualTo("second"); + } + + @Test + void samplingDoesNotCacheContextOrOutput() { + var values = WeightedYamlValues.from(List.of(Map.of("60%", "first"), Map.of("40%", "second")), UnaryOperator.identity()); + assertThat(values.select(fixed(0, 0))).isEqualTo("first"); + assertThat(values.select(fixed(0.99, 0))).isEqualTo("second"); + assertThat(values.select(fixed(0, 0))).isEqualTo("first"); + RandomService a = new RandomService(new Random(123)); + RandomService b = new RandomService(new Random(123)); + for (int i = 0; i < 100; i++) assertThat(values.select(a)).isEqualTo(values.select(b)); + } + + @Test + void qualifiesBothBranchShapesWithoutChangingSource() { + List source = List.of(Map.of("60%", "#{suffix}"), Map.of("40%", List.of("#{other}"))); + var values = WeightedYamlValues.from(source, s -> s.replace("#{", "#{Custom.")); + assertThat(values.select(fixed(0, 0))).isEqualTo("#{Custom.suffix}"); + assertThat(values.select(fixed(0.99, 0))).isEqualTo("#{Custom.other}"); + assertThat(source.get(0)).isEqualTo(Map.of("60%", "#{suffix}")); + } + + @Test + void sumsDecimalPercentagesExactly() { + var values = WeightedYamlValues.from(List.of(Map.of("33.3%", "a"), Map.of("33.3%", "b"), Map.of("33.4%", "c")), UnaryOperator.identity()); + assertThat(values.select(fixed(0.99, 0))).isEqualTo("c"); + } + + @ParameterizedTest + @MethodSource("invalidChoices") + void rejectsMalformedChoices(List choices) { + assertThatThrownBy(() -> WeightedYamlValues.from(choices, UnaryOperator.identity())) + .isInstanceOf(IllegalArgumentException.class).hasMessageStartingWith("Invalid weighted YAML choices:"); + } + + static Stream> invalidChoices() { + return Stream.of( + List.of(Map.of("60%", "a"), "b"), + List.of(Map.of("60%", "a"), Map.of("plain", "b")), + List.of(Map.of("60%", "a"), Map.of()), + List.of(Map.of("60%", "a", "40%", "b")), + List.of(Map.of("0%", "a"), Map.of("100%", "b")), + List.of(Map.of("-1%", "a"), Map.of("101%", "b")), + List.of(Map.of("NaN%", "a")), + List.of(Map.of("Infinity%", "a")), + List.of(Map.of("60 %", "a")), + List.of(Map.of("60%", "a"), Map.of("30%", "b")), + List.of(Map.of("60%", "a"), Map.of("50%", "b")), + List.of(Map.of("100%", List.of())), + List.of(Map.of("100%", List.of("a", 1))), + List.of(Map.of("100%", Map.of("nested", "a"))), + List.of(Map.of("100%", List.of(Map.of("100%", "a")))), + List.of(Map.of("60%", "same"), Map.of("40%", "same")) + ); + } + + private static RandomService fixed(double draw, int index) { + return new RandomService(new Random(0) { + @Override + public double nextDouble() { + return draw; + } + + @Override + public int nextInt(int bound) { + if (index < 0 || index >= bound) throw new AssertionError("Unexpected list size " + bound); + return index; + } + }); + } +} diff --git a/src/test/resources/weighted-custom.yml b/src/test/resources/weighted-custom.yml new file mode 100644 index 000000000..bc0e719ee --- /dev/null +++ b/src/test/resources/weighted-custom.yml @@ -0,0 +1,26 @@ +en: + faker: + weighted_custom: + strings: + - "60%": "first" + - "40%": "second" + domains: + - "90%": ".it" + - "10%": + - ".ee" + - ".lv" + - ".lt" + local_template: + - "100%": "#{suffix}!" + suffix: + - "chosen" + reference: + - "100%": "#{Name.first_name}" + ordinary: + - "a" + - "b" + empty: [] + scalar: "plain" + nested: + value: + - "100%": "nested"