| 1 |
/* |
| 2 |
* Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 |
* contributor license agreements. See the NOTICE file distributed with |
| 4 |
* this work for additional information regarding copyright ownership. |
| 5 |
* The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 |
* (the "License"); you may not use this file except in compliance with |
| 7 |
* the License. You may obtain a copy of the License at |
| 8 |
* |
| 9 |
* http://www.apache.org/licenses/LICENSE-2.0 |
| 10 |
* |
| 11 |
* Unless required by applicable law or agreed to in writing, software |
| 12 |
* distributed under the License is distributed on an "AS IS" BASIS, |
| 13 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 |
* See the License for the specific language governing permissions and |
| 15 |
* limitations under the License. |
| 16 |
*/ |
| 17 |
package org.apache.lucene.gradle.datasets; |
| 18 |
|
| 19 |
import java.io.BufferedReader; |
| 20 |
import java.io.BufferedWriter; |
| 21 |
import java.io.IOException; |
| 22 |
import java.nio.charset.StandardCharsets; |
| 23 |
import java.nio.file.DirectoryStream; |
| 24 |
import java.nio.file.Files; |
| 25 |
import java.nio.file.Path; |
| 26 |
import java.nio.file.Paths; |
| 27 |
import java.nio.file.StandardCopyOption; |
| 28 |
import java.util.regex.Matcher; |
| 29 |
import java.util.regex.Pattern; |
| 30 |
import java.util.stream.Stream; |
| 31 |
|
| 32 |
/** |
| 33 |
* Split the Reuters SGML documents into Simple Text files containing: |
| 34 |
* Title, Date, Dateline, Body |
| 35 |
*/ |
| 36 |
public class ExtractReuters { |
| 37 |
private Path reutersDir; |
| 38 |
private Path outputDir; |
| 39 |
|
| 40 |
public ExtractReuters(Path reutersDir, Path outputDir) throws IOException { |
| 41 |
this.reutersDir = reutersDir; |
| 42 |
this.outputDir = outputDir; |
| 43 |
} |
| 44 |
|
| 45 |
public void extract() throws IOException { |
| 46 |
long count = 0; |
| 47 |
Files.createDirectories(outputDir); |
| 48 |
try(Stream<Path> files = Files.list(outputDir)) { |
| 49 |
if (files.count() > 0) { |
| 50 |
throw new IOException("The output directory must be empty: " + outputDir); |
| 51 |
} |
| 52 |
} |
| 53 |
|
| 54 |
try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) { |
| 55 |
for (Path sgmFile : stream) { |
| 56 |
extractFile(sgmFile); |
| 57 |
count++; |
| 58 |
} |
| 59 |
} |
| 60 |
if (count == 0) { |
| 61 |
throw new IOException("No .sgm files in " + reutersDir); |
| 62 |
} |
| 63 |
} |
| 64 |
|
| 65 |
Pattern EXTRACTION_PATTERN = |
| 66 |
Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>"); |
| 67 |
|
| 68 |
private static String[] META_CHARS = {"&", "<", ">", "\"", "'"}; |
| 69 |
|
| 70 |
private static String[] META_CHARS_SERIALIZATIONS = {"&", "<", ">", """, "'"}; |
| 71 |
|
| 72 |
/** Override if you wish to change what is extracted */ |
| 73 |
protected void extractFile(Path sgmFile) throws IOException { |
| 74 |
try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.ISO_8859_1)) { |
| 75 |
StringBuilder buffer = new StringBuilder(1024); |
| 76 |
StringBuilder outBuffer = new StringBuilder(1024); |
| 77 |
|
| 78 |
String line = null; |
| 79 |
int docNumber = 0; |
| 80 |
while ((line = reader.readLine()) != null) { |
| 81 |
// when we see a closing reuters tag, flush the file |
| 82 |
|
| 83 |
if (line.indexOf("</REUTERS") == -1) { |
| 84 |
// Replace the SGM escape sequences |
| 85 |
|
| 86 |
buffer.append(line).append(' '); // accumulate the strings for now, |
| 87 |
// then apply regular expression to |
| 88 |
// get the pieces, |
| 89 |
} else { |
| 90 |
// Extract the relevant pieces and write to a file in the output dir |
| 91 |
Matcher matcher = EXTRACTION_PATTERN.matcher(buffer); |
| 92 |
while (matcher.find()) { |
| 93 |
for (int i = 1; i <= matcher.groupCount(); i++) { |
| 94 |
if (matcher.group(i) != null) { |
| 95 |
outBuffer.append(matcher.group(i)); |
| 96 |
} |
| 97 |
} |
| 98 |
outBuffer.append(System.lineSeparator()).append(System.lineSeparator()); |
| 99 |
} |
| 100 |
String out = outBuffer.toString(); |
| 101 |
for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) { |
| 102 |
out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]); |
| 103 |
} |
| 104 |
Path outFile = outputDir.resolve(sgmFile.getFileName() + "-" + (docNumber++) + ".txt"); |
| 105 |
// System.out.println("Writing " + outFile); |
| 106 |
try (BufferedWriter writer = Files.newBufferedWriter(outFile, StandardCharsets.UTF_8)) { |
| 107 |
writer.write(out); |
| 108 |
} |
| 109 |
outBuffer.setLength(0); |
| 110 |
buffer.setLength(0); |
| 111 |
} |
| 112 |
} |
| 113 |
} |
| 114 |
} |
| 115 |
|
| 116 |
public static void main(String[] args) throws Exception { |
| 117 |
if (args.length != 2) { |
| 118 |
usage("Wrong number of arguments (" + args.length + ")"); |
| 119 |
return; |
| 120 |
} |
| 121 |
Path reutersDir = Paths.get(args[0]); |
| 122 |
if (!Files.exists(reutersDir)) { |
| 123 |
usage("Cannot find Path to Reuters SGM files (" + reutersDir + ")"); |
| 124 |
return; |
| 125 |
} |
| 126 |
|
| 127 |
// First, extract to a tmp directory and only if everything succeeds, rename |
| 128 |
// to output directory. |
| 129 |
Path outputDir = Paths.get(args[1] + "-tmp"); |
| 130 |
Files.createDirectories(outputDir); |
| 131 |
ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir); |
| 132 |
extractor.extract(); |
| 133 |
// Now rename to requested output dir |
| 134 |
Files.move(outputDir, Paths.get(args[1]), StandardCopyOption.ATOMIC_MOVE); |
| 135 |
} |
| 136 |
|
| 137 |
private static void usage(String msg) { |
| 138 |
System.err.println( |
| 139 |
"Usage: " |
| 140 |
+ msg |
| 141 |
+ " :: java -cp <...> " |
| 142 |
+ ExtractReuters.class.getName() |
| 143 |
+ " <Path to Reuters SGM files> <Output Path>"); |
| 144 |
} |
| 145 |
} |