Debian Codegraph @

buildSrc/src/main/java/org/apache/lucene/gradle/datasets/ExtractReuters.java

raw
1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17 package org.apache.lucene.gradle.datasets;
18
19 import java.io.BufferedReader;
20 import java.io.BufferedWriter;
21 import java.io.IOException;
22 import java.nio.charset.StandardCharsets;
23 import java.nio.file.DirectoryStream;
24 import java.nio.file.Files;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.nio.file.StandardCopyOption;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30 import java.util.stream.Stream;
31
32 /**
33 * Split the Reuters SGML documents into Simple Text files containing:
34 * Title, Date, Dateline, Body
35 */
36 public class ExtractReuters {
37 private Path reutersDir;
38 private Path outputDir;
39
40 public ExtractReuters(Path reutersDir, Path outputDir) throws IOException {
41 this.reutersDir = reutersDir;
42 this.outputDir = outputDir;
43 }
44
45 public void extract() throws IOException {
46 long count = 0;
47 Files.createDirectories(outputDir);
48 try(Stream<Path> files = Files.list(outputDir)) {
49 if (files.count() > 0) {
50 throw new IOException("The output directory must be empty: " + outputDir);
51 }
52 }
53
54 try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) {
55 for (Path sgmFile : stream) {
56 extractFile(sgmFile);
57 count++;
58 }
59 }
60 if (count == 0) {
61 throw new IOException("No .sgm files in " + reutersDir);
62 }
63 }
64
65 Pattern EXTRACTION_PATTERN =
66 Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
67
68 private static String[] META_CHARS = {"&", "<", ">", "\"", "'"};
69
70 private static String[] META_CHARS_SERIALIZATIONS = {"&amp;", "&lt;", "&gt;", "&quot;", "&apos;"};
71
72 /** Override if you wish to change what is extracted */
73 protected void extractFile(Path sgmFile) throws IOException {
74 try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.ISO_8859_1)) {
75 StringBuilder buffer = new StringBuilder(1024);
76 StringBuilder outBuffer = new StringBuilder(1024);
77
78 String line = null;
79 int docNumber = 0;
80 while ((line = reader.readLine()) != null) {
81 // when we see a closing reuters tag, flush the file
82
83 if (line.indexOf("</REUTERS") == -1) {
84 // Replace the SGM escape sequences
85
86 buffer.append(line).append(' '); // accumulate the strings for now,
87 // then apply regular expression to
88 // get the pieces,
89 } else {
90 // Extract the relevant pieces and write to a file in the output dir
91 Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
92 while (matcher.find()) {
93 for (int i = 1; i <= matcher.groupCount(); i++) {
94 if (matcher.group(i) != null) {
95 outBuffer.append(matcher.group(i));
96 }
97 }
98 outBuffer.append(System.lineSeparator()).append(System.lineSeparator());
99 }
100 String out = outBuffer.toString();
101 for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
102 out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
103 }
104 Path outFile = outputDir.resolve(sgmFile.getFileName() + "-" + (docNumber++) + ".txt");
105 // System.out.println("Writing " + outFile);
106 try (BufferedWriter writer = Files.newBufferedWriter(outFile, StandardCharsets.UTF_8)) {
107 writer.write(out);
108 }
109 outBuffer.setLength(0);
110 buffer.setLength(0);
111 }
112 }
113 }
114 }
115
116 public static void main(String[] args) throws Exception {
117 if (args.length != 2) {
118 usage("Wrong number of arguments (" + args.length + ")");
119 return;
120 }
121 Path reutersDir = Paths.get(args[0]);
122 if (!Files.exists(reutersDir)) {
123 usage("Cannot find Path to Reuters SGM files (" + reutersDir + ")");
124 return;
125 }
126
127 // First, extract to a tmp directory and only if everything succeeds, rename
128 // to output directory.
129 Path outputDir = Paths.get(args[1] + "-tmp");
130 Files.createDirectories(outputDir);
131 ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
132 extractor.extract();
133 // Now rename to requested output dir
134 Files.move(outputDir, Paths.get(args[1]), StandardCopyOption.ATOMIC_MOVE);
135 }
136
137 private static void usage(String msg) {
138 System.err.println(
139 "Usage: "
140 + msg
141 + " :: java -cp <...> "
142 + ExtractReuters.class.getName()
143 + " <Path to Reuters SGM files> <Output Path>");
144 }
145 }