| 1 | package com.renomad.minum.database; | |
| 2 | ||
| 3 | import com.renomad.minum.queue.AbstractActionQueue; | |
| 4 | import com.renomad.minum.queue.ActionQueue; | |
| 5 | import com.renomad.minum.state.Context; | |
| 6 | ||
| 7 | import java.io.BufferedReader; | |
| 8 | import java.io.FileReader; | |
| 9 | import java.io.IOException; | |
| 10 | import java.nio.charset.StandardCharsets; | |
| 11 | import java.nio.file.Files; | |
| 12 | import java.nio.file.Path; | |
| 13 | import java.util.Collection; | |
| 14 | import java.util.Collections; | |
| 15 | import java.util.concurrent.atomic.AtomicLong; | |
| 16 | import java.util.concurrent.locks.ReentrantLock; | |
| 17 | import java.util.function.Function; | |
| 18 | ||
| 19 | import static com.renomad.minum.utils.Invariants.mustBeFalse; | |
| 20 | import static com.renomad.minum.utils.Invariants.mustBeTrue; | |
| 21 | ||
| 22 | /** | |
| 23 | * a memory-based disk-persisted database class. | |
| 24 | * @param <T> the type of data we'll be persisting (must extend from {@link DbData} | |
| 25 | */ | |
| 26 | public class Db<T extends DbData<?>> extends AbstractDb<T> { | |
| 27 | ||
| 28 | /** | |
| 29 | * The suffix we will apply to each database file | |
| 30 | */ | |
| 31 | static final String DATABASE_FILE_SUFFIX = ".ddps"; | |
| 32 | private final AbstractActionQueue actionQueue; | |
| 33 | private final ReentrantLock loadDataLock = new ReentrantLock(); | |
| 34 | ||
| 35 | /** | |
| 36 | * The full path to the file that contains the most-recent index | |
| 37 | * for this data. As we add new files, each gets its own index | |
| 38 | * value. When we start the program, we use this to determine | |
| 39 | * where to start counting for new indexes. | |
| 40 | */ | |
| 41 | private final Path fullPathForIndexFile; | |
| 42 | ||
| 43 | /** | |
| 44 | * An in-memory representation of the value of the current max index | |
| 45 | * that we store in index.ddps, in memory, so we can compare whether | |
| 46 | * we need to update the disk without checking the disk so often. | |
| 47 | */ | |
| 48 | private long maxIndexOnDisk; | |
| 49 | ||
| 50 | private boolean hasLoadedData; | |
| 51 | ||
| 52 | /** | |
| 53 | * Constructs an in-memory disk-persisted database. | |
| 54 | * Loading of data from disk happens at the first invocation of any command | |
| 55 | * changing or requesting data, such as {@link #write(DbData)}, {@link #delete(DbData)}, | |
| 56 | * or {@link #values()}. See the private method loadData() for details. | |
| 57 | * @param dbDirectory this uniquely names your database, and also sets the directory | |
| 58 | * name for this data. The expected use case is to name this after | |
| 59 | * the data in question. For example, "users", or "accounts". | |
| 60 | * @param context used to provide important state data to several components | |
| 61 | * @param instance an instance of the {@link DbData} object relevant for use in this database. Note | |
| 62 | * that each database (that is, each instance of this class), focuses on just one | |
| 63 | * data, which must be an implementation of {@link DbData}. | |
| 64 | */ | |
| 65 | public Db(Path dbDirectory, Context context, T instance) { | |
| 66 | super(dbDirectory, context, instance); | |
| 67 | this.hasLoadedData = false; | |
| 68 | this.fullPathForIndexFile = dbDirectory.resolve("index" + DATABASE_FILE_SUFFIX); | |
| 69 | this.actionQueue = new ActionQueue("DatabaseWriter " + dbDirectory, context).initialize(); | |
| 70 | ||
| 71 |
1
1. <init> : negated conditional → KILLED |
if (Files.exists(fullPathForIndexFile)) { |
| 72 | long indexValue; | |
| 73 | try (var fileReader = new FileReader(fullPathForIndexFile.toFile(), StandardCharsets.UTF_8)) { | |
| 74 | try (BufferedReader br = new BufferedReader(fileReader)) { | |
| 75 | String s = br.readLine(); | |
| 76 |
1
1. <init> : negated conditional → KILLED |
if (s == null) throw new DbException("index file at " + fullPathForIndexFile + " returned null when reading a line from it"); |
| 77 | mustBeFalse(s.isBlank(), "Unless something is terribly broken, we expect a numeric value here"); | |
| 78 | String trim = s.trim(); | |
| 79 | indexValue = Long.parseLong(trim); | |
| 80 | } | |
| 81 | } catch (Exception e) { | |
| 82 | throw new DbException("Exception while reading "+fullPathForIndexFile+" in Db constructor", e); | |
| 83 | } | |
| 84 | ||
| 85 | this.index = new AtomicLong(indexValue); | |
| 86 | ||
| 87 | } else { | |
| 88 | this.index = new AtomicLong(1); | |
| 89 | } | |
| 90 | ||
| 91 |
2
1. lambda$new$0 : removed call to com/renomad/minum/utils/FileUtils::makeDirectory → KILLED 2. <init> : removed call to com/renomad/minum/queue/AbstractActionQueue::enqueue → KILLED |
actionQueue.enqueue("create directory" + dbDirectory, () -> fileUtils.makeDirectory(dbDirectory)); |
| 92 | } | |
| 93 | ||
| 94 | /** | |
| 95 | * Write data to the database. Use an index of 0 to store new data, and a positive | |
| 96 | * non-zero value to update data. | |
| 97 | * <p><em> | |
| 98 | * Example of adding new data to the database: | |
| 99 | * </p></em> | |
| 100 | * {@snippet : | |
| 101 | * final var newSalt = StringUtils.generateSecureRandomString(10); | |
| 102 | * final var hashedPassword = CryptoUtils.createPasswordHash(newPassword, newSalt); | |
| 103 | * final var newUser = new User(0L, newUsername, hashedPassword, newSalt); | |
| 104 | * userDb.write(newUser); | |
| 105 | * } | |
| 106 | * <p><em> | |
| 107 | * Example of updating data: | |
| 108 | * </p></em> | |
| 109 | * {@snippet : | |
| 110 | * // write the updated salted password to the database | |
| 111 | * final var updatedUser = new User( | |
| 112 | * user().getIndex(), | |
| 113 | * user().getUsername(), | |
| 114 | * hashedPassword, | |
| 115 | * newSalt); | |
| 116 | * userDb.write(updatedUser); | |
| 117 | * } | |
| 118 | * | |
| 119 | * @param newData the data we are writing | |
| 120 | * @return the data with its new index assigned. | |
| 121 | */ | |
| 122 | @Override | |
| 123 | public T write(T newData) { | |
| 124 |
2
1. write : negated conditional → KILLED 2. write : changed conditional boundary → KILLED |
if (newData.getIndex() < 0) throw new DbException("Negative indexes are disallowed"); |
| 125 | // load data if needed | |
| 126 |
2
1. write : negated conditional → KILLED 2. write : removed call to com/renomad/minum/database/Db::loadData → KILLED |
if (!hasLoadedData) loadData(); |
| 127 | ||
| 128 | boolean newElementCreated = processDataIndex(newData); | |
| 129 |
1
1. write : removed call to com/renomad/minum/database/Db::writeToMemory → KILLED |
writeToMemory(newData, newElementCreated); |
| 130 | ||
| 131 | // *** now handle the disk portion *** | |
| 132 |
2
1. write : removed call to com/renomad/minum/queue/AbstractActionQueue::enqueue → KILLED 2. lambda$write$1 : removed call to com/renomad/minum/database/Db::writeToDisk → KILLED |
actionQueue.enqueue("persist data to disk", () -> writeToDisk(newData)); |
| 133 | ||
| 134 | // returning the data at this point is the most convenient | |
| 135 | // way users will have access to the new index of the data. | |
| 136 |
1
1. write : replaced return value with null for com/renomad/minum/database/Db::write → KILLED |
return newData; |
| 137 | } | |
| 138 | ||
| 139 | private void writeToDisk(T newData) { | |
| 140 | final Path fullPath = dbDirectory.resolve(newData.getIndex() + DATABASE_FILE_SUFFIX); | |
| 141 | logger.logTrace(() -> String.format("writing data to %s", fullPath)); | |
| 142 | String serializedData = newData.serialize(); | |
| 143 | mustBeFalse(serializedData == null || serializedData.isBlank(), | |
| 144 | "the serialized form of data must not be blank. " + | |
| 145 | "Is the serialization code written properly? Our datatype: " + emptyInstance); | |
| 146 |
1
1. writeToDisk : removed call to com/renomad/minum/utils/FileUtils::writeString → KILLED |
fileUtils.writeString(fullPath, serializedData); |
| 147 |
2
1. writeToDisk : negated conditional → KILLED 2. writeToDisk : changed conditional boundary → KILLED |
if (maxIndexOnDisk < index.get()) { |
| 148 | maxIndexOnDisk = index.get(); | |
| 149 |
1
1. writeToDisk : removed call to com/renomad/minum/utils/FileUtils::writeString → KILLED |
fileUtils.writeString(fullPathForIndexFile, String.valueOf(maxIndexOnDisk)); |
| 150 | } | |
| 151 | } | |
| 152 | ||
| 153 | /** | |
| 154 | * Delete data | |
| 155 | * <p><em>Example:</p></em> | |
| 156 | * {@snippet : | |
| 157 | * userDb.delete(user); | |
| 158 | * } | |
| 159 | * @param dataToDelete the data we are serializing and writing | |
| 160 | */ | |
| 161 | @Override | |
| 162 | public void delete(T dataToDelete) { | |
| 163 | // load data if needed | |
| 164 |
2
1. delete : negated conditional → KILLED 2. delete : removed call to com/renomad/minum/database/Db::loadData → KILLED |
if (!hasLoadedData) loadData(); |
| 165 | ||
| 166 | // deal with the in-memory portion | |
| 167 |
1
1. delete : removed call to com/renomad/minum/database/Db::deleteFromMemory → KILLED |
deleteFromMemory(dataToDelete); |
| 168 | ||
| 169 | // now handle the disk portion | |
| 170 |
2
1. lambda$delete$3 : removed call to com/renomad/minum/database/Db::deleteFromDisk → KILLED 2. delete : removed call to com/renomad/minum/queue/AbstractActionQueue::enqueue → KILLED |
actionQueue.enqueue("delete data from disk", () -> deleteFromDisk(dataToDelete)); |
| 171 | } | |
| 172 | ||
| 173 | private void deleteFromDisk(T dataToDelete) { | |
| 174 | final Path fullPath = dbDirectory.resolve(dataToDelete.getIndex() + DATABASE_FILE_SUFFIX); | |
| 175 | logger.logTrace(() -> String.format("deleting data at %s", fullPath)); | |
| 176 | try { | |
| 177 |
1
1. deleteFromDisk : negated conditional → KILLED |
if (!fullPath.toFile().exists()) { |
| 178 | throw new DbException(fullPath + " must already exist before deletion"); | |
| 179 | } | |
| 180 |
1
1. deleteFromDisk : removed call to java/nio/file/Files::delete → KILLED |
Files.delete(fullPath); |
| 181 |
2
1. deleteFromDisk : changed conditional boundary → KILLED 2. deleteFromDisk : negated conditional → KILLED |
if (maxIndexOnDisk > index.get()) { |
| 182 | maxIndexOnDisk = index.get(); | |
| 183 |
1
1. deleteFromDisk : removed call to com/renomad/minum/utils/FileUtils::writeString → KILLED |
fileUtils.writeString(fullPathForIndexFile, String.valueOf(maxIndexOnDisk)); |
| 184 | ||
| 185 | } | |
| 186 | } catch (Exception ex) { | |
| 187 | logger.logAsyncError(() -> "failed to delete file " + fullPath + " during deleteOnDisk. Exception: " + ex); | |
| 188 | } | |
| 189 | } | |
| 190 | ||
| 191 | /** | |
| 192 | * Grabs all the data from disk and returns it as a list. This | |
| 193 | * method is run by various programs when the system first loads. | |
| 194 | */ | |
| 195 | private void loadDataFromDisk() throws IOException { | |
| 196 | // check if the folder has content for a DbEngine2 database, meaning we | |
| 197 | // need to convert it back to the classic DB file structure. | |
| 198 |
1
1. loadDataFromDisk : negated conditional → KILLED |
if (Files.exists(dbDirectory.resolve("currentAppendLog"))) { |
| 199 |
1
1. loadDataFromDisk : removed call to com/renomad/minum/database/DbFileConverter::convertFolderStructureToDbClassic → KILLED |
new DbFileConverter(context, dbDirectory).convertFolderStructureToDbClassic(); |
| 200 | } | |
| 201 | ||
| 202 |
1
1. loadDataFromDisk : negated conditional → KILLED |
if (! Files.exists(dbDirectory)) { |
| 203 | logger.logDebug(() -> dbDirectory + " directory missing, adding nothing to the data list"); | |
| 204 | return; | |
| 205 | } | |
| 206 | ||
| 207 |
1
1. loadDataFromDisk : removed call to com/renomad/minum/database/Db::walkAndLoad → KILLED |
walkAndLoad(dbDirectory); |
| 208 | } | |
| 209 | ||
| 210 | void walkAndLoad(Path dbDirectory) { | |
| 211 | // walk through all the files in this directory, collecting | |
| 212 | // all regular files (non-subdirectories) except for index.ddps | |
| 213 | try (final var pathStream = Files.walk(dbDirectory)) { | |
| 214 | final var listOfFiles = pathStream.filter(path -> | |
| 215 |
2
1. lambda$walkAndLoad$7 : replaced boolean return with true for com/renomad/minum/database/Db::lambda$walkAndLoad$7 → KILLED 2. lambda$walkAndLoad$7 : negated conditional → KILLED |
Files.isRegularFile(path) && |
| 216 |
1
1. lambda$walkAndLoad$7 : negated conditional → KILLED |
!path.getFileName().toString().startsWith("index") |
| 217 | ).toList(); | |
| 218 | for (Path p : listOfFiles) { | |
| 219 |
1
1. walkAndLoad : removed call to com/renomad/minum/database/Db::readAndDeserialize → KILLED |
readAndDeserialize(p); |
| 220 | } | |
| 221 | } catch (IOException e) { | |
| 222 | throw new DbException(e); | |
| 223 | } | |
| 224 | } | |
| 225 | ||
| 226 | /** | |
| 227 | * Carry out the process of reading data files into our in-memory structure | |
| 228 | * @param p the path of a particular file | |
| 229 | */ | |
| 230 | void readAndDeserialize(Path p) throws IOException { | |
| 231 | Path fileName = p.getFileName(); | |
| 232 |
1
1. readAndDeserialize : negated conditional → KILLED |
if (fileName == null) throw new DbException("At readAndDeserialize, path " + p + " returned a null filename"); |
| 233 | String filename = fileName.toString(); | |
| 234 | int startOfSuffixIndex = filename.indexOf('.'); | |
| 235 |
1
1. readAndDeserialize : negated conditional → KILLED |
if(startOfSuffixIndex == -1) { |
| 236 | throw new DbException("the files must have a ddps suffix, like 1.ddps. filename: " + filename); | |
| 237 | } | |
| 238 | String fileContents = Files.readString(p); | |
| 239 |
1
1. readAndDeserialize : negated conditional → KILLED |
if (fileContents.isBlank()) { |
| 240 | logger.logDebug( () -> fileName + " file exists but empty, skipping"); | |
| 241 | } else { | |
| 242 | try { | |
| 243 | @SuppressWarnings("unchecked") | |
| 244 | T deserializedData = (T) emptyInstance.deserialize(fileContents); | |
| 245 | mustBeTrue(deserializedData != null, "deserialization of " + emptyInstance + | |
| 246 | " resulted in a null value. Was the serialization method implemented properly?"); | |
| 247 | int fileNameIdentifier = Integer.parseInt(filename.substring(0, startOfSuffixIndex)); | |
| 248 | mustBeTrue(deserializedData.getIndex() == fileNameIdentifier, | |
| 249 | "The filename must correspond to the data's index. e.g. 1.ddps must have an id of 1"); | |
| 250 | ||
| 251 | // put the data into the in-memory data structure | |
| 252 | data.put(deserializedData.getIndex(), deserializedData); | |
| 253 |
1
1. readAndDeserialize : removed call to com/renomad/minum/database/Db::addToIndexes → KILLED |
addToIndexes(deserializedData); |
| 254 | ||
| 255 | } catch (Exception e) { | |
| 256 | throw new DbException("Failed to deserialize "+ p +" with data (\""+fileContents+"\"). Caused by: " + e); | |
| 257 | } | |
| 258 | } | |
| 259 | } | |
| 260 | ||
| 261 | @Override | |
| 262 | public Collection<T> values() { | |
| 263 | // load data if needed | |
| 264 |
2
1. values : negated conditional → KILLED 2. values : removed call to com/renomad/minum/database/Db::loadData → KILLED |
if (!hasLoadedData) loadData(); |
| 265 | ||
| 266 |
1
1. values : replaced return value with Collections.emptyList for com/renomad/minum/database/Db::values → KILLED |
return Collections.unmodifiableCollection(data.values()); |
| 267 | } | |
| 268 | ||
| 269 | /** | |
| 270 | * This is what loads the data from disk the | |
| 271 | * first time someone needs it. Because it is | |
| 272 | * locked, only one thread can enter at | |
| 273 | * a time. The first one in will load the data, | |
| 274 | * and the second will encounter a branch which skips loading. | |
| 275 | */ | |
| 276 | @Override | |
| 277 | public void loadData() { | |
| 278 |
1
1. loadData : removed call to java/util/concurrent/locks/ReentrantLock::lock → KILLED |
loadDataLock.lock(); // block threads here if multiple are trying to get in - only one gets in at a time |
| 279 | try { | |
| 280 |
1
1. loadData : negated conditional → KILLED |
if (!hasLoadedData) { |
| 281 |
1
1. loadData : removed call to com/renomad/minum/database/Db::loadDataFromDisk → KILLED |
loadDataFromDisk(); |
| 282 | } | |
| 283 | hasLoadedData = true; | |
| 284 | } catch (Exception ex) { | |
| 285 | throw new DbException("Failed to load data from disk.", ex); | |
| 286 | } finally { | |
| 287 |
1
1. loadData : removed call to java/util/concurrent/locks/ReentrantLock::unlock → KILLED |
loadDataLock.unlock(); |
| 288 | } | |
| 289 | } | |
| 290 | ||
| 291 | /** | |
| 292 | * Register an index in the database for higher performance data access. | |
| 293 | * <p> | |
| 294 | * This command should be run immediately after database declaration, | |
| 295 | * or more specifically, before any data is loaded from disk. Otherwise, | |
| 296 | * it would be possible to skip indexing that data. | |
| 297 | * </p> | |
| 298 | * <br> | |
| 299 | * Example: | |
| 300 | * {@snippet : | |
| 301 | * final var myDatabase = context.getDb("photos", Photograph.EMPTY); | |
| 302 | * myDatabase.registerIndex("url", photo -> photo.getUrl()); | |
| 303 | * } | |
| 304 | * @param indexName a string used to distinguish this index. This string will be used again | |
| 305 | * when requesting data in a method like {@link #getIndexedData} or {@link #findExactlyOne} | |
| 306 | * @param keyObtainingFunction a function which obtains data from the data in this database, used | |
| 307 | * to partition the data into groups (potentially up to a 1-to-1 correspondence | |
| 308 | * between id and object) | |
| 309 | * @return true if the registration succeeded | |
| 310 | * @throws DbException if the parameters are not entered properly, if the index has already | |
| 311 | * been registered, or if the data has already been loaded. It is necessary that | |
| 312 | * this is run immediately after declaring the database. To explain further: the data is not | |
| 313 | * actually loaded until the first time it is needed, such as running a write or delete, or | |
| 314 | * if the {@link #loadDataFromDisk()} method is run. Creating an index map for the data that | |
| 315 | * is read from disk only occurs once, at data load time. Thus, it is crucial that the | |
| 316 | * registerIndex command is run before any data is loaded. | |
| 317 | */ | |
| 318 | @Override | |
| 319 | public boolean registerIndex(String indexName, Function<T, String> keyObtainingFunction) { | |
| 320 |
1
1. registerIndex : negated conditional → KILLED |
if (hasLoadedData) { |
| 321 | throw new DbException("This method must be run before the database loads data from disk. Typically, " + | |
| 322 | "it should be run immediately after the database is created. See this method's documentation"); | |
| 323 | } | |
| 324 |
2
1. registerIndex : replaced boolean return with true for com/renomad/minum/database/Db::registerIndex → SURVIVED 2. registerIndex : replaced boolean return with false for com/renomad/minum/database/Db::registerIndex → KILLED |
return super.registerIndex(indexName, keyObtainingFunction); |
| 325 | } | |
| 326 | ||
| 327 | /** | |
| 328 | * Given the name of a registered index (see {@link #registerIndex(String, Function)}), | |
| 329 | * use the key to find the collection of data that matches it. | |
| 330 | * @param indexName the name of an index | |
| 331 | * @param key a string value that matches a partition calculated from the partition | |
| 332 | * function provided to {@link #registerIndex(String, Function)} | |
| 333 | * @return a collection of data, an empty collection if nothing found | |
| 334 | */ | |
| 335 | @Override | |
| 336 | public Collection<T> getIndexedData(String indexName, String key) { | |
| 337 | // load data if needed | |
| 338 |
2
1. getIndexedData : negated conditional → KILLED 2. getIndexedData : removed call to com/renomad/minum/database/Db::loadData → KILLED |
if (!hasLoadedData) loadData(); |
| 339 |
1
1. getIndexedData : replaced return value with Collections.emptyList for com/renomad/minum/database/Db::getIndexedData → KILLED |
return super.getIndexedData(indexName, key); |
| 340 | } | |
| 341 | ||
| 342 | /** | |
| 343 | * This function will stop the minum.database persistence cleanly. | |
| 344 | * <p> | |
| 345 | * In order to do this, we need to wait for our threads | |
| 346 | * to finish their work. In particular, we | |
| 347 | * have offloaded our file writes to [actionQueue], which | |
| 348 | * has an internal thread for serializing all actions | |
| 349 | * on our minum.database | |
| 350 | * </p> | |
| 351 | */ | |
| 352 | @Override | |
| 353 | public void stop() { | |
| 354 |
1
1. stop : removed call to com/renomad/minum/queue/AbstractActionQueue::stop → KILLED |
actionQueue.stop(); |
| 355 | } | |
| 356 | ||
| 357 | /** | |
| 358 | * Similar to {@link #stop()} but gives more control over how long | |
| 359 | * we'll wait before crashing it closed. See {@link ActionQueue#stop(int, int)} | |
| 360 | */ | |
| 361 | @Override | |
| 362 | public void stop(int count, int sleepTime) { | |
| 363 |
1
1. stop : removed call to com/renomad/minum/queue/AbstractActionQueue::stop → KILLED |
actionQueue.stop(count, sleepTime); |
| 364 | } | |
| 365 | ||
| 366 | } | |
Mutations | ||
| 71 |
1.1 |
|
| 76 |
1.1 |
|
| 91 |
1.1 2.2 |
|
| 124 |
1.1 2.2 |
|
| 126 |
1.1 2.2 |
|
| 129 |
1.1 |
|
| 132 |
1.1 2.2 |
|
| 136 |
1.1 |
|
| 146 |
1.1 |
|
| 147 |
1.1 2.2 |
|
| 149 |
1.1 |
|
| 164 |
1.1 2.2 |
|
| 167 |
1.1 |
|
| 170 |
1.1 2.2 |
|
| 177 |
1.1 |
|
| 180 |
1.1 |
|
| 181 |
1.1 2.2 |
|
| 183 |
1.1 |
|
| 198 |
1.1 |
|
| 199 |
1.1 |
|
| 202 |
1.1 |
|
| 207 |
1.1 |
|
| 215 |
1.1 2.2 |
|
| 216 |
1.1 |
|
| 219 |
1.1 |
|
| 232 |
1.1 |
|
| 235 |
1.1 |
|
| 239 |
1.1 |
|
| 253 |
1.1 |
|
| 264 |
1.1 2.2 |
|
| 266 |
1.1 |
|
| 278 |
1.1 |
|
| 280 |
1.1 |
|
| 281 |
1.1 |
|
| 287 |
1.1 |
|
| 320 |
1.1 |
|
| 324 |
1.1 2.2 |
|
| 338 |
1.1 2.2 |
|
| 339 |
1.1 |
|
| 354 |
1.1 |
|
| 363 |
1.1 |