Debian Codegraph @

test-data/pkg/writer/geoip2.go

raw
1 package writer
2
3 import (
4 "encoding/json"
5 "fmt"
6 "net/netip"
7 "os"
8 "path/filepath"
9 "strings"
10
11 "github.com/maxmind/mmdbwriter"
12 "github.com/maxmind/mmdbwriter/mmdbtype"
13 "go4.org/netipx"
14 )
15
16 // WriteGeoIP2TestDB writes GeoIP2 test mmdb files.
17 func (w *Writer) WriteGeoIP2TestDB() error {
18 // This is a map from database type to input file name.
19 dbTypes := map[string]string{
20 "GeoIP-Anonymous-Plus": "GeoIP-Anonymous-Plus-Test.json",
21 "GeoIP2-Anonymous-IP": "GeoIP2-Anonymous-IP-Test.json",
22 "GeoIP2-City": "GeoIP2-City-Test.json",
23 "GeoIP2-City-Shield": "GeoIP2-City-Test.json",
24 "GeoIP2-Connection-Type": "GeoIP2-Connection-Type-Test.json",
25 "GeoIP2-Country": "GeoIP2-Country-Test.json",
26 "GeoIP2-Country-Shield": "GeoIP2-Country-Test.json",
27 "GeoIP2-DensityIncome": "GeoIP2-DensityIncome-Test.json",
28 "GeoIP2-Domain": "GeoIP2-Domain-Test.json",
29 "GeoIP2-Enterprise": "GeoIP2-Enterprise-Test.json",
30 "GeoIP2-Enterprise-Shield": "GeoIP2-Enterprise-Test.json",
31 "GeoIP2-IP-Risk": "GeoIP2-IP-Risk-Test.json",
32 "GeoIP2-ISP": "GeoIP2-ISP-Test.json",
33 "GeoIP2-Precision-Enterprise": "GeoIP2-Precision-Enterprise-Test.json",
34 "GeoIP2-Precision-Enterprise-Shield": "GeoIP2-Precision-Enterprise-Test.json",
35 "GeoIP2-Static-IP-Score": "GeoIP2-Static-IP-Score-Test.json",
36 "GeoIP2-User-Count": "GeoIP2-User-Count-Test.json",
37 "GeoLite2-ASN": "GeoLite2-ASN-Test.json",
38 "GeoLite2-City": "GeoLite2-City-Test.json",
39 "GeoLite2-Country": "GeoLite2-Country-Test.json",
40 }
41
42 for dbType, jsonFileName := range dbTypes {
43 languages := []string{"en"}
44 description := map[string]string{
45 "en": strings.ReplaceAll(dbType, "-", " ") +
46 " Test Database (fake GeoIP2 data, for example purposes only)",
47 }
48
49 if dbType == "GeoIP2-City" {
50 languages = append(languages, "zh")
51 description["zh"] = "小型数据库"
52 }
53
54 dbWriter, err := mmdbwriter.New(
55 mmdbwriter.Options{
56 DatabaseType: dbType,
57 Description: description,
58 DisableIPv4Aliasing: false,
59 IPVersion: 6,
60 Languages: languages,
61 RecordSize: 28,
62 },
63 )
64 if err != nil {
65 return fmt.Errorf("creating mmdbwriter: %w", err)
66 }
67
68 if dbType == "GeoIP2-Anonymous-IP" || dbType == "GeoIP-Anonymous-Plus" {
69 if err := populateAllNetworks(dbWriter); err != nil {
70 return fmt.Errorf("inserting all networks: %w", err)
71 }
72 }
73
74 if err := InsertJSON(dbWriter, filepath.Join(w.source, jsonFileName)); err != nil {
75 return fmt.Errorf("inserting json: %w", err)
76 }
77
78 dbFileName := dbType + "-Test.mmdb"
79 if err := w.write(dbWriter, dbFileName); err != nil {
80 return fmt.Errorf("writing database: %w", err)
81 }
82 }
83
84 return nil
85 }
86
87 // InsertJSON reads and parses a json file into mmdbtypes values and inserts
88 // them into the mmdbwriter tree.
89 func InsertJSON(dbWriter *mmdbwriter.Tree, filePath string) error {
90 file, err := os.Open(filepath.Clean(filePath))
91 if err != nil {
92 return fmt.Errorf("opening json file: %w", err)
93 }
94 defer file.Close()
95
96 var data []map[string]any
97 if err := json.NewDecoder(file).Decode(&data); err != nil {
98 return fmt.Errorf("decoding json file: %w", err)
99 }
100
101 for _, record := range data {
102 for k, v := range record {
103 prefix, err := netip.ParsePrefix(k)
104 if err != nil {
105 return fmt.Errorf("parsing ip: %w", err)
106 }
107
108 mmdbValue, err := toMMDBType(prefix.String(), v)
109 if err != nil {
110 return fmt.Errorf("converting value to mmdbtype: %w", err)
111 }
112
113 err = dbWriter.Insert(
114 netipx.PrefixIPNet(prefix),
115 mmdbValue,
116 )
117 if err != nil {
118 return fmt.Errorf("inserting ip: %w", err)
119 }
120 }
121 }
122 return nil
123 }
124
125 // toMMDBType key converts field values read from json into their corresponding mmdbtype.DataType.
126 // It makes some assumptions for numeric types based on previous knowledge about field types.
127 func toMMDBType(key string, value any) (mmdbtype.DataType, error) {
128 switch v := value.(type) {
129 case bool:
130 return mmdbtype.Bool(v), nil
131 case string:
132 return mmdbtype.String(v), nil
133 case map[string]any:
134 m := mmdbtype.Map{}
135 for innerKey, val := range v {
136 innerVal, err := toMMDBType(innerKey, val)
137 if err != nil {
138 return nil, fmt.Errorf("parsing mmdbtype.Map for key %q: %w", key, err)
139 }
140 m[mmdbtype.String(innerKey)] = innerVal
141 }
142 return m, nil
143 case []any:
144 s := mmdbtype.Slice{}
145 for _, val := range v {
146 innerVal, err := toMMDBType(key, val)
147 if err != nil {
148 return nil, fmt.Errorf("parsing mmdbtype.Slice for key %q: %w", key, err)
149 }
150 s = append(s, innerVal)
151 }
152 return s, nil
153 case float64:
154 switch key {
155 case "accuracy_radius", "anonymizer_confidence", "confidence", "metro_code":
156 return mmdbtype.Uint16(v), nil
157 case "autonomous_system_number", "average_income",
158 "geoname_id", "ipv4_24", "ipv4_32", "ipv6_32",
159 "ipv6_48", "ipv6_64", "population_density":
160 return mmdbtype.Uint32(v), nil
161 case "ip_risk", "latitude", "longitude", "score",
162 "static_ip_score":
163 return mmdbtype.Float64(v), nil
164 default:
165 return nil, fmt.Errorf("unsupported numeric type for key %q: %T", key, value)
166 }
167 default:
168 return nil, fmt.Errorf("unsupported type for key %q: %T", key, value)
169 }
170 }
171
172 // populate all networks inserts all networks into the writer with an empty map value.
173 func populateAllNetworks(w *mmdbwriter.Tree) error {
174 defaultNet, err := netip.ParsePrefix("::/0")
175 if err != nil {
176 return fmt.Errorf("parsing ip: %w", err)
177 }
178
179 err = w.Insert(
180 netipx.PrefixIPNet(defaultNet),
181 mmdbtype.Map{},
182 )
183 if err != nil {
184 return fmt.Errorf("inserting ip: %w", err)
185 }
186
187 return nil
188 }