| 1 | package com.renomad.minum.database; | |
| 2 | ||
| 3 | import com.renomad.minum.logging.ILogger; | |
| 4 | import com.renomad.minum.state.Context; | |
| 5 | ||
| 6 | import java.io.BufferedReader; | |
| 7 | import java.io.FileReader; | |
| 8 | import java.io.IOException; | |
| 9 | import java.nio.file.Files; | |
| 10 | import java.nio.file.Path; | |
| 11 | import java.util.ArrayList; | |
| 12 | import java.util.Comparator; | |
| 13 | import java.util.List; | |
| 14 | import java.util.stream.Stream; | |
| 15 | ||
| 16 | import static com.renomad.minum.utils.Invariants.mustBeFalse; | |
| 17 | import static java.nio.charset.StandardCharsets.US_ASCII; | |
| 18 | import static java.nio.file.StandardOpenOption.*; | |
| 19 | ||
| 20 | /** | |
| 21 | * This class exists to handle converting from one file/folder | |
| 22 | * style of database to another. | |
| 23 | */ | |
| 24 | final class DbFileConverter { | |
| 25 | ||
| 26 | private final Path dbDirectory; | |
| 27 | private final ILogger logger; | |
| 28 | private final DatabaseAppender databaseAppender; | |
| 29 | private final DatabaseConsolidator databaseConsolidator; | |
| 30 | ||
| 31 | /** | |
| 32 | * Construct a converter instance | |
| 33 | * @param context this is used for its constants, logger, and so on. | |
| 34 | * @param dbDirectory this is the specific directory for this database, | |
| 35 | * for example, the full path to the "users" database directory. | |
| 36 | */ | |
| 37 | DbFileConverter(Context context, Path dbDirectory) throws IOException { | |
| 38 | this.dbDirectory = dbDirectory; | |
| 39 | this.logger = context.getLogger(); | |
| 40 | this.databaseAppender = new DatabaseAppender(dbDirectory, context); | |
| 41 | this.databaseConsolidator = new DatabaseConsolidator(dbDirectory, context); | |
| 42 | } | |
| 43 | ||
| 44 | /** | |
| 45 | * convert a directory of database files from the old Db classic form to the DbEngine2 form. | |
| 46 | * The old form was one file per data item, the new form has append-only | |
| 47 | * data logs and consolidated files. | |
| 48 | */ | |
| 49 | void convertClassicFolderStructureToDbEngine2Form() throws IOException { | |
| 50 |
1
1. convertClassicFolderStructureToDbEngine2Form : removed call to com/renomad/minum/database/DbFileConverter::displayWarningConvertingClassicToDbEngine2 → TIMED_OUT |
displayWarningConvertingClassicToDbEngine2(); |
| 51 | try (var fileReader = new FileReader(dbDirectory.resolve("index.ddps").toFile(), US_ASCII)) { | |
| 52 | try (BufferedReader br = new BufferedReader(fileReader)) { | |
| 53 | String s = br.readLine(); | |
| 54 |
1
1. convertClassicFolderStructureToDbEngine2Form : negated conditional → KILLED |
if (s == null) throw new DbException("index file for " + dbDirectory + " returned null when reading a line from it"); |
| 55 | mustBeFalse(s.isBlank(), "Unless something is terribly broken, we expect a numeric value here"); | |
| 56 | } | |
| 57 | } | |
| 58 | ||
| 59 |
1
1. convertClassicFolderStructureToDbEngine2Form : removed call to com/renomad/minum/database/DbFileConverter::walkFilesAndConvertDbToDbEngine2 → KILLED |
walkFilesAndConvertDbToDbEngine2(this.dbDirectory, this.logger); |
| 60 | } | |
| 61 | ||
| 62 | ||
| 63 | ||
| 64 | private void displayWarningConvertingClassicToDbEngine2() { | |
| 65 | logger.logDebug(() -> "*****************************************************************"); | |
| 66 | logger.logDebug(() -> "*****************************************************************"); | |
| 67 | logger.logDebug(() -> "........"); | |
| 68 | logger.logDebug(() -> "About to convert database files from Db Classic to Db Engine 2"); | |
| 69 | logger.logDebug(() -> "........"); | |
| 70 | logger.logDebug(() -> "*****************************************************************"); | |
| 71 | logger.logDebug(() -> "*****************************************************************"); | |
| 72 | } | |
| 73 | ||
| 74 | private void displayWarningConvertingDbEngine2ToClassic() { | |
| 75 | logger.logDebug(() -> "*****************************************************************"); | |
| 76 | logger.logDebug(() -> "*****************************************************************"); | |
| 77 | logger.logDebug(() -> "........"); | |
| 78 | logger.logDebug(() -> "About to convert database files from Db Engine2 to Db Classic"); | |
| 79 | logger.logDebug(() -> "........"); | |
| 80 | logger.logDebug(() -> "*****************************************************************"); | |
| 81 | logger.logDebug(() -> "*****************************************************************"); | |
| 82 | } | |
| 83 | ||
| 84 | /** | |
| 85 | * walk through all the files in this directory, collecting | |
| 86 | * all regular files (non-subdirectories) except for index.ddps | |
| 87 | */ | |
| 88 | static void walkFilesAndConvertDbToDbEngine2(Path dbDirectory, ILogger logger) throws IOException { | |
| 89 | List<Path> listOfFiles = getListOfFiles(dbDirectory); | |
| 90 | ||
| 91 | // convert each file to the new database schema by appending it | |
| 92 | // to the append log, and then delete it | |
| 93 |
2
1. walkFilesAndConvertDbToDbEngine2 : negated conditional → KILLED 2. walkFilesAndConvertDbToDbEngine2 : changed conditional boundary → KILLED |
for (int i = 0; i < listOfFiles.size(); i++) { |
| 94 |
1
1. walkFilesAndConvertDbToDbEngine2 : Replaced integer division with multiplication → TIMED_OUT |
int percentCompletion = listOfFiles.size() / 100; |
| 95 |
1
1. walkFilesAndConvertDbToDbEngine2 : negated conditional → TIMED_OUT |
if (i == percentCompletion) { |
| 96 | logger.logDebug(() -> "File converting is %d percent complete".formatted(percentCompletion)); | |
| 97 | } | |
| 98 | Path p = extractDataAndAppend(dbDirectory, logger, listOfFiles.get(i)); | |
| 99 |
1
1. walkFilesAndConvertDbToDbEngine2 : removed call to java/nio/file/Files::delete → KILLED |
Files.delete(p); |
| 100 | } | |
| 101 | ||
| 102 | // at this point, after all the ordinary files have been removed, kill the index file | |
| 103 |
1
1. walkFilesAndConvertDbToDbEngine2 : removed call to java/nio/file/Files::delete → TIMED_OUT |
Files.delete(dbDirectory.resolve("index.ddps")); |
| 104 | } | |
| 105 | ||
| 106 | /** | |
| 107 | * This method digs into the Db Classic file, checks everything is kosher, and | |
| 108 | * if so, appends it to the append-only log file which is part of DbEngine2 | |
| 109 | */ | |
| 110 | static Path extractDataAndAppend(Path dbDirectory, ILogger logger, Path fileToAnalyze) throws IOException { | |
| 111 | String fileContents = checkFileDetailsAreValid(fileToAnalyze, logger); | |
| 112 |
1
1. extractDataAndAppend : negated conditional → KILLED |
if (!fileContents.isBlank()) { |
| 113 | Files.writeString( | |
| 114 | dbDirectory.resolve("currentAppendLog"), | |
| 115 | "UPDATE %s\n".formatted(fileContents), APPEND, CREATE); | |
| 116 | } | |
| 117 |
1
1. extractDataAndAppend : replaced return value with null for com/renomad/minum/database/DbFileConverter::extractDataAndAppend → KILLED |
return fileToAnalyze; |
| 118 | } | |
| 119 | ||
| 120 | /** | |
| 121 | * Get the files that make up the file schema of Db Classic | |
| 122 | */ | |
| 123 | private static List<Path> getListOfFiles(Path dbDirectory) { | |
| 124 | List<Path> listOfFiles; | |
| 125 | try (Stream<Path> fileStream = Files.list(dbDirectory)) { | |
| 126 | listOfFiles = fileStream.filter(path -> | |
| 127 |
2
1. lambda$getListOfFiles$15 : replaced boolean return with true for com/renomad/minum/database/DbFileConverter::lambda$getListOfFiles$15 → KILLED 2. lambda$getListOfFiles$15 : negated conditional → KILLED |
Files.isRegularFile(path) && |
| 128 |
1
1. lambda$getListOfFiles$15 : negated conditional → KILLED |
path.getFileName().toString().endsWith(".ddps") && |
| 129 |
1
1. lambda$getListOfFiles$15 : negated conditional → KILLED |
!path.getFileName().toString().startsWith("index")) |
| 130 | .toList(); | |
| 131 | } catch (IOException ex) { | |
| 132 | throw new DbException("Failed during the listing of files during conversion of db to db engine2", ex); | |
| 133 | } | |
| 134 |
1
1. getListOfFiles : replaced return value with Collections.emptyList for com/renomad/minum/database/DbFileConverter::getListOfFiles → KILLED |
return listOfFiles; |
| 135 | } | |
| 136 | ||
| 137 | /** | |
| 138 | * This code inspects that the data in the file is valid | |
| 139 | * and returns the data. This method is called as part of | |
| 140 | * convert Db Classic to DbEngine2 | |
| 141 | */ | |
| 142 | static String checkFileDetailsAreValid(Path p, ILogger logger) throws IOException { | |
| 143 |
1
1. checkFileDetailsAreValid : negated conditional → KILLED |
if (!Files.isRegularFile(p)) { |
| 144 | throw new DbException("At checkFileDetailsAreValid, path " + p + " is not a regular file"); | |
| 145 | } | |
| 146 | String fileName = p.getFileName().toString(); | |
| 147 | int startOfSuffixIndex = fileName.indexOf('.'); | |
| 148 | String fileContents = Files.readString(p); | |
| 149 |
1
1. checkFileDetailsAreValid : negated conditional → KILLED |
if (fileContents.isBlank()) { |
| 150 | logger.logDebug( () -> fileName + " file exists but empty, skipping"); | |
| 151 | return ""; | |
| 152 | } else { | |
| 153 |
1
1. checkFileDetailsAreValid : replaced return value with "" for com/renomad/minum/database/DbFileConverter::checkFileDetailsAreValid → TIMED_OUT |
return deserializeDataFromDbFile(fileName, startOfSuffixIndex, fileContents); |
| 154 | } | |
| 155 | } | |
| 156 | ||
| 157 | /** | |
| 158 | * While converting Db Classic files to DbEngine2, each data file will be inspected. | |
| 159 | * Here, we are looking at the contents of the files. | |
| 160 | */ | |
| 161 | static String deserializeDataFromDbFile(String filename, int startOfSuffixIndex, String fileContents) { | |
| 162 | int fileNameIdentifier = Integer.parseInt(filename.substring(0, startOfSuffixIndex)); | |
| 163 | int indexOfFirstPipe = fileContents.indexOf('|'); | |
| 164 | String indexString = fileContents.substring(0, indexOfFirstPipe); | |
| 165 | long index = Long.parseLong(indexString); | |
| 166 |
1
1. deserializeDataFromDbFile : negated conditional → KILLED |
if (index != fileNameIdentifier) { |
| 167 | throw new DbException( "The filename (%s) must correspond to the index in its contents (%d)" | |
| 168 | .formatted(filename, index)); | |
| 169 | } | |
| 170 |
1
1. deserializeDataFromDbFile : replaced return value with "" for com/renomad/minum/database/DbFileConverter::deserializeDataFromDbFile → KILLED |
return fileContents; |
| 171 | } | |
| 172 | ||
| 173 | /** | |
| 174 | * Convert the folder/file structure. From DbEngine2 format to Db classic. | |
| 175 | */ | |
| 176 | void convertFolderStructureToDbClassic() throws IOException { | |
| 177 |
1
1. convertFolderStructureToDbClassic : removed call to com/renomad/minum/database/DbFileConverter::displayWarningConvertingDbEngine2ToClassic → TIMED_OUT |
displayWarningConvertingDbEngine2ToClassic(); |
| 178 | ||
| 179 | // if there are any remnant items in the current append-only file, move them | |
| 180 | // to a new file | |
| 181 | databaseAppender.saveOffCurrentDataToReadyFolder(); | |
| 182 | ||
| 183 | // consolidate whatever files still exist in the append logs | |
| 184 |
1
1. convertFolderStructureToDbClassic : removed call to com/renomad/minum/database/DatabaseConsolidator::consolidate → KILLED |
databaseConsolidator.consolidate(); |
| 185 | ||
| 186 | // at this point, all the data is consolidated, in order, so we can step | |
| 187 | // through the data, creating new files, and finish up with an index.ddps | |
| 188 | // set to the proper value (i.e. one greater than the max index in the database) | |
| 189 |
1
1. convertFolderStructureToDbClassic : removed call to com/renomad/minum/database/DbFileConverter::walkFilesAndConvertDbEngine2ToDbClassic → KILLED |
walkFilesAndConvertDbEngine2ToDbClassic(this.dbDirectory, this.logger); |
| 190 | } | |
| 191 | ||
| 192 | /** | |
| 193 | * This is a pretty intricate method. It basically steps through the consolidated | |
| 194 | * data files, writing each line to a new file. It has to handle this through | |
| 195 | * multiple files and deleting files when finished, and closing file handles | |
| 196 | * appropriately when done with a file, and doing it in the proper order. | |
| 197 | */ | |
| 198 | static void walkFilesAndConvertDbEngine2ToDbClassic(Path dbDirectory, ILogger logger) throws IOException { | |
| 199 | // get the list of consolidated data files | |
| 200 | List<Path> listOfFiles; | |
| 201 | try (Stream<Path> fileStream = Files.list(dbDirectory.resolve("consolidated_data"))) { | |
| 202 |
2
1. lambda$walkFilesAndConvertDbEngine2ToDbClassic$17 : replaced boolean return with true for com/renomad/minum/database/DbFileConverter::lambda$walkFilesAndConvertDbEngine2ToDbClassic$17 → TIMED_OUT 2. lambda$walkFilesAndConvertDbEngine2ToDbClassic$17 : replaced boolean return with false for com/renomad/minum/database/DbFileConverter::lambda$walkFilesAndConvertDbEngine2ToDbClassic$17 → KILLED |
listOfFiles = new ArrayList<>(fileStream.filter(Files::isRegularFile).toList()); |
| 203 | } catch (IOException ex) { | |
| 204 | throw new DbException("Failed during the listing of files during conversion of db engine2 to db classic", ex); | |
| 205 | } | |
| 206 | ||
| 207 | // sort the data in ascending order. Files like "1_to_10" will come before "11_to_20" | |
| 208 | listOfFiles.sort(Comparator.comparing(x -> { | |
| 209 | String filename = x.getFileName().toString(); | |
| 210 | int indexOfFirstUnderscore = filename.indexOf('_'); | |
| 211 |
1
1. lambda$walkFilesAndConvertDbEngine2ToDbClassic$18 : negated conditional → KILLED |
if (indexOfFirstUnderscore == -1) { |
| 212 | throw new DbException("Error: Failed to find first underscore in filename: " + filename); | |
| 213 | } | |
| 214 | String firstIndexNumberOfFile = filename.substring(0, indexOfFirstUnderscore); | |
| 215 | try { | |
| 216 |
1
1. lambda$walkFilesAndConvertDbEngine2ToDbClassic$18 : replaced Long return value with 0L for com/renomad/minum/database/DbFileConverter::lambda$walkFilesAndConvertDbEngine2ToDbClassic$18 → SURVIVED |
return Long.parseLong(firstIndexNumberOfFile); |
| 217 | } catch (NumberFormatException ex) { | |
| 218 | throw new DbException("Failed to convert first part of filename to a number: " + filename); | |
| 219 | } | |
| 220 | })); | |
| 221 | ||
| 222 | // initialize a variable to record the current maximum index value | |
| 223 | long currentMaxIndexValue = -1; | |
| 224 | ||
| 225 | // initialize a variable to record the count of data items converted to Db Classic | |
| 226 | long countConvertedFiles = 0; | |
| 227 | ||
| 228 | // convert each line of each file to its own file, per the needs of Db Classic | |
| 229 | for (Path filePath : listOfFiles) { | |
| 230 | try (BufferedReader reader = Files.newBufferedReader(filePath.toFile().toPath(), US_ASCII)) { | |
| 231 | String line; | |
| 232 | while ((line = reader.readLine()) != null) { | |
| 233 | int i = line.indexOf('|'); | |
| 234 | if (i == -1) { | |
| 235 | throw new DbException(("Unable to convert a line - check for " + | |
| 236 | "corruption. File: %s Data: %s").formatted(filePath, line)); | |
| 237 | } | |
| 238 | String indexNumberString = line.substring(0, i); | |
| 239 | long indexNumber; | |
| 240 | try { | |
| 241 | indexNumber = Long.parseLong(indexNumberString); | |
| 242 | currentMaxIndexValue = Math.max(indexNumber, currentMaxIndexValue); | |
| 243 | } catch (NumberFormatException ex) { | |
| 244 | throw new DbException(("Unable to convert a line - check for " + | |
| 245 | "corruption. File: %s Data: %s").formatted(filePath, line)); | |
| 246 | } | |
| 247 | String dbFilename = indexNumber + ".ddps"; | |
| 248 | Path dbFullPath = dbDirectory.resolve(dbFilename); | |
| 249 | Files.writeString(dbFullPath, line, CREATE, WRITE); | |
| 250 | countConvertedFiles += 1; | |
| 251 | logAlongConversion(logger, countConvertedFiles, 1000); | |
| 252 | } | |
| 253 | } | |
| 254 | // now we've converted everything from this file, delete it. | |
| 255 |
1
1. walkFilesAndConvertDbEngine2ToDbClassic : removed call to java/nio/file/Files::delete → KILLED |
Files.delete(filePath); |
| 256 | } | |
| 257 | ||
| 258 |
1
1. walkFilesAndConvertDbEngine2ToDbClassic : removed call to com/renomad/minum/database/DbFileConverter::deleteEmptyDbEngine2Directories → TIMED_OUT |
deleteEmptyDbEngine2Directories(dbDirectory); |
| 259 |
1
1. walkFilesAndConvertDbEngine2ToDbClassic : removed call to com/renomad/minum/database/DbFileConverter::createNewIndexFile → KILLED |
createNewIndexFile(dbDirectory, currentMaxIndexValue); |
| 260 | } | |
| 261 | ||
| 262 | /** | |
| 263 | * This small helper method will create an `index.ddps` file with the correct value | |
| 264 | * for use with Db classic. It determines the correct value by having calculated the | |
| 265 | * maximum-value-seen throughout the conversion process. | |
| 266 | */ | |
| 267 | static void createNewIndexFile(Path dbDirectory, long currentMaxIndexValue) { | |
| 268 | // create an index.ddps with a value set to the current max, plus one | |
| 269 | try { | |
| 270 |
1
1. createNewIndexFile : Replaced long addition with subtraction → KILLED |
Files.writeString(dbDirectory.resolve("index.ddps"), String.valueOf(currentMaxIndexValue + 1), CREATE_NEW); |
| 271 | } catch (IOException ex) { | |
| 272 | throw new DbException("Failed to create an index.ddps file", ex); | |
| 273 | } | |
| 274 | } | |
| 275 | ||
| 276 | /** | |
| 277 | * This method is called after conversion to Db classic, at which point these directories | |
| 278 | * will be empty. | |
| 279 | */ | |
| 280 | static void deleteEmptyDbEngine2Directories(Path dbDirectory) { | |
| 281 | try { | |
| 282 | Files.deleteIfExists(dbDirectory.resolve("consolidated_data")); | |
| 283 | Files.deleteIfExists(dbDirectory.resolve("currentAppendLog")); | |
| 284 | Files.deleteIfExists(dbDirectory.resolve("append_logs")); | |
| 285 | } catch (IOException ex) { | |
| 286 | throw new DbException("Failed to delete one of the DbEngine2 files", ex); | |
| 287 | } | |
| 288 | } | |
| 289 | ||
| 290 | /** | |
| 291 | * A helper to choose when to output logging statements during the conversion | |
| 292 | * of files from one database file format to another. | |
| 293 | * @param countConvertedFilesModulo modulo this number at which a log statement will be output. | |
| 294 | */ | |
| 295 | static void logAlongConversion(ILogger logger, long countConvertedFiles, int countConvertedFilesModulo) { | |
| 296 |
2
1. logAlongConversion : Replaced long modulus with multiplication → KILLED 2. logAlongConversion : negated conditional → KILLED |
if (countConvertedFiles % countConvertedFilesModulo == 0) { |
| 297 | logger.logDebug(() -> "DbFileConverter has converted %d files to Db Classic form" | |
| 298 | .formatted(countConvertedFiles)); | |
| 299 | } | |
| 300 | } | |
| 301 | } | |
Mutations | ||
| 50 |
1.1 |
|
| 54 |
1.1 |
|
| 59 |
1.1 |
|
| 93 |
1.1 2.2 |
|
| 94 |
1.1 |
|
| 95 |
1.1 |
|
| 99 |
1.1 |
|
| 103 |
1.1 |
|
| 112 |
1.1 |
|
| 117 |
1.1 |
|
| 127 |
1.1 2.2 |
|
| 128 |
1.1 |
|
| 129 |
1.1 |
|
| 134 |
1.1 |
|
| 143 |
1.1 |
|
| 149 |
1.1 |
|
| 153 |
1.1 |
|
| 166 |
1.1 |
|
| 170 |
1.1 |
|
| 177 |
1.1 |
|
| 184 |
1.1 |
|
| 189 |
1.1 |
|
| 202 |
1.1 2.2 |
|
| 211 |
1.1 |
|
| 216 |
1.1 |
|
| 255 |
1.1 |
|
| 258 |
1.1 |
|
| 259 |
1.1 |
|
| 270 |
1.1 |
|
| 296 |
1.1 2.2 |