| 1 | package com.renomad.minum.database; | |
| 2 | ||
| 3 | ||
| 4 | import com.renomad.minum.logging.ILogger; | |
| 5 | import com.renomad.minum.state.Constants; | |
| 6 | import com.renomad.minum.state.Context; | |
| 7 | import com.renomad.minum.utils.FileUtils; | |
| 8 | import com.renomad.minum.utils.MyThread; | |
| 9 | import com.renomad.minum.utils.StacktraceUtils; | |
| 10 | ||
| 11 | import java.io.BufferedWriter; | |
| 12 | import java.io.IOException; | |
| 13 | import java.io.Writer; | |
| 14 | import java.nio.charset.StandardCharsets; | |
| 15 | import java.nio.file.Files; | |
| 16 | import java.nio.file.Path; | |
| 17 | import java.nio.file.StandardOpenOption; | |
| 18 | import java.text.SimpleDateFormat; | |
| 19 | import java.util.List; | |
| 20 | import java.util.concurrent.ExecutorService; | |
| 21 | import java.util.concurrent.locks.ReentrantLock; | |
| 22 | ||
| 23 | /** | |
| 24 | * This class provide the capability of appending database changes | |
| 25 | * to the disk, quickly and efficiently. | |
| 26 | */ | |
| 27 | final class DatabaseAppender { | |
| 28 | ||
| 29 | /** | |
| 30 | * Results in output like "2025_08_30_13_01_49_123", which is year_month_day_hour_minute_second_millisecond. | |
| 31 | * This can be used to parse the file names to {@link java.util.Date} so we can process the oldest | |
| 32 | * file first. | |
| 33 | */ | |
| 34 | static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS"); | |
| 35 | ||
| 36 | private final Path persistenceDirectory; | |
| 37 | ||
| 38 | Writer bufferedWriter; | |
| 39 | ||
| 40 | /** | |
| 41 | * if true, there is data in the buffered writer that needs to be | |
| 42 | * written to disk using {@link BufferedWriter#flush()} | |
| 43 | */ | |
| 44 | private boolean bufferedWriterHasUnwrittenData; | |
| 45 | ||
| 46 | /** | |
| 47 | * This class field tracks the status of the loop which runs | |
| 48 | * a flush every second | |
| 49 | */ | |
| 50 | private boolean flushLoopRunning; | |
| 51 | ||
| 52 | /** | |
| 53 | * The directory for this database | |
| 54 | */ | |
| 55 | private final Path appendLogDirectory; | |
| 56 | ||
| 57 | /** | |
| 58 | * Used to create a thread that contains an inner loop | |
| 59 | * to flush the data to disk on a periodic basis | |
| 60 | */ | |
| 61 | private final ExecutorService executorService; | |
| 62 | ||
| 63 | private final ILogger logger; | |
| 64 | ||
| 65 | private final ReentrantLock moveFileLock; | |
| 66 | ||
| 67 | /** | |
| 68 | * The maximum number of data's we will add to the append-only | |
| 69 | * file before we move on to a new file. | |
| 70 | */ | |
| 71 | int maxAppendCount; | |
| 72 | ||
| 73 | /** | |
| 74 | * this is the current count of how many appends have | |
| 75 | * been made to the current database file. Once it | |
| 76 | * exceeds a certain maximum, we'll switch to a | |
| 77 | * different file. | |
| 78 | */ | |
| 79 | int appendCount; | |
| 80 | ||
| 81 | /** | |
| 82 | * This is the count of bytes that have been appended | |
| 83 | */ | |
| 84 | private long appendBytes; | |
| 85 | ||
| 86 | DatabaseAppender(Path persistenceDirectory, Context context) throws IOException { | |
| 87 | this.persistenceDirectory = persistenceDirectory; | |
| 88 | this.appendLogDirectory = persistenceDirectory.resolve("append_logs"); | |
| 89 | this.executorService = context.getExecutorService(); | |
| 90 | this.logger = context.getLogger(); | |
| 91 | Constants constants = context.getConstants(); | |
| 92 | FileUtils fileUtils = new FileUtils(logger, constants); | |
| 93 | this.maxAppendCount = constants.maxAppendCount; | |
| 94 |
1
1. <init> : removed call to com/renomad/minum/utils/FileUtils::makeDirectory → KILLED |
fileUtils.makeDirectory(this.appendLogDirectory); |
| 95 | moveFileLock = new ReentrantLock(); | |
| 96 |
1
1. <init> : removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED |
createNewAppendFile(); |
| 97 | } | |
| 98 | ||
| 99 | /** | |
| 100 | * Creates a new append-file (a file used for appending data) and | |
| 101 | * resets the append count to zero. | |
| 102 | */ | |
| 103 | private void createNewAppendFile() throws IOException { | |
| 104 | Path currentAppendFile = this.persistenceDirectory.resolve("currentAppendLog"); | |
| 105 | ||
| 106 | // if we are starting up with an existing currentAppendLog, set the appendCount | |
| 107 | // appropriately. Otherwise, initialize to 0. The currentAppendLog file is | |
| 108 | // never very large - it's mostly a temporary place to store incoming data | |
| 109 | // until we can store it off elsewhere. For that reason, it's not a performance | |
| 110 | // concern to read all the existing lines, just to get the count of current lines. | |
| 111 |
1
1. createNewAppendFile : negated conditional → KILLED |
if (Files.exists(currentAppendFile)) { |
| 112 | List<String> lines = Files.readAllLines(currentAppendFile); | |
| 113 | appendCount = lines.size(); | |
| 114 | } else { | |
| 115 | // reset the count to zero, we're starting a new file. | |
| 116 | logger.logDebug(() -> "Creating a new database append file. Previous file: %,d lines, %.2f megabytes".formatted(appendCount, (appendBytes / 1_048_576.0))); | |
| 117 | appendCount = 0; | |
| 118 | appendBytes = 0; | |
| 119 | } | |
| 120 | ||
| 121 | bufferedWriter = Files.newBufferedWriter(currentAppendFile, StandardCharsets.US_ASCII, StandardOpenOption.CREATE, StandardOpenOption.APPEND); | |
| 122 | } | |
| 123 | ||
| 124 | /** | |
| 125 | * Appends new data to the end of a file. | |
| 126 | * @return if we created a new append file, we'll return the name of it. Otherwise, an empty string. | |
| 127 | */ | |
| 128 | String appendToDatabase(DatabaseChangeAction action, String serializedData) throws IOException { | |
| 129 | String newlyCreatedFileName = ""; | |
| 130 |
2
1. appendToDatabase : changed conditional boundary → KILLED 2. appendToDatabase : negated conditional → KILLED |
if (appendCount >= maxAppendCount) { |
| 131 |
1
1. appendToDatabase : removed call to java/util/concurrent/locks/ReentrantLock::lock → KILLED |
moveFileLock.lock(); // block threads here if multiple are trying to get in - only one gets in at a time |
| 132 | try { | |
| 133 | newlyCreatedFileName = saveOffWrapped(appendCount, maxAppendCount); | |
| 134 | } finally { | |
| 135 |
1
1. appendToDatabase : removed call to java/util/concurrent/locks/ReentrantLock::unlock → TIMED_OUT |
moveFileLock.unlock(); |
| 136 | } | |
| 137 | } | |
| 138 | ||
| 139 | bufferedWriter.append(action.toString()).append(' ').append(serializedData).append('\n'); | |
| 140 |
1
1. appendToDatabase : removed call to com/renomad/minum/database/DatabaseAppender::setBufferedWriterHasUnwrittenData → TIMED_OUT |
setBufferedWriterHasUnwrittenData(); |
| 141 |
1
1. appendToDatabase : Replaced integer addition with subtraction → SURVIVED |
appendCount += 1; |
| 142 |
2
1. appendToDatabase : Replaced long addition with subtraction → SURVIVED 2. appendToDatabase : Replaced integer addition with subtraction → TIMED_OUT |
appendBytes += serializedData.length() + 8; // 8 includes the action (e.g. UPDATE), a space character, and a newline |
| 143 |
1
1. appendToDatabase : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::appendToDatabase → KILLED |
return newlyCreatedFileName; |
| 144 | } | |
| 145 | ||
| 146 | private void setBufferedWriterHasUnwrittenData() { | |
| 147 | bufferedWriterHasUnwrittenData = true; | |
| 148 |
1
1. setBufferedWriterHasUnwrittenData : negated conditional → SURVIVED |
if (!flushLoopRunning) { |
| 149 |
1
1. setBufferedWriterHasUnwrittenData : removed call to com/renomad/minum/database/DatabaseAppender::initializeTimedFlusher → TIMED_OUT |
initializeTimedFlusher(); |
| 150 | } | |
| 151 | } | |
| 152 | ||
| 153 | /** | |
| 154 | * This method is kicked off when there is new data added to | |
| 155 | * the {@link BufferedWriter}. While there is data to write, it | |
| 156 | * will wake up every second to flush the data. Once there is | |
| 157 | * no more data, it will end. | |
| 158 | */ | |
| 159 | private void initializeTimedFlusher() { | |
| 160 | Runnable timedFlusherLoop = () -> { | |
| 161 | flushLoopRunning = true; | |
| 162 | Thread.currentThread().setName("database_timed_flusher"); | |
| 163 |
1
1. lambda$initializeTimedFlusher$1 : negated conditional → TIMED_OUT |
while (bufferedWriterHasUnwrittenData) { |
| 164 |
1
1. lambda$initializeTimedFlusher$1 : removed call to com/renomad/minum/database/DatabaseAppender::flush → TIMED_OUT |
flush(); |
| 165 | ||
| 166 | // this code only runs when there is data to add, so no need to take a | |
| 167 | // lot of waiting time. But, if the data is coming fast and furious, | |
| 168 | // at least a small wait will allow greater efficiency. | |
| 169 | MyThread.sleep(50); | |
| 170 | } | |
| 171 | flushLoopRunning = false; | |
| 172 | }; | |
| 173 | executorService.submit(timedFlusherLoop); | |
| 174 | } | |
| 175 | ||
| 176 | /** | |
| 177 | * This helper just wraps a method to enable easier testing. | |
| 178 | * @return true if the appendCount is greater or equal to maxAppendCount, | |
| 179 | * meaning that we moved on to calling {@link #saveOffCurrentDataToReadyFolder()}, | |
| 180 | * false otherwise. | |
| 181 | */ | |
| 182 | String saveOffWrapped(int appendCount, int maxAppendCount) throws IOException { | |
| 183 |
2
1. saveOffWrapped : changed conditional boundary → KILLED 2. saveOffWrapped : negated conditional → KILLED |
if (appendCount >= maxAppendCount) { |
| 184 |
1
1. saveOffWrapped : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffWrapped → KILLED |
return saveOffCurrentDataToReadyFolder(); |
| 185 | } | |
| 186 | return ""; | |
| 187 | } | |
| 188 | ||
| 189 | /** | |
| 190 | * Move the append-only file to a new place to prepare for | |
| 191 | * consolidation, and reset the append count. | |
| 192 | * @return the name of the newly-created file | |
| 193 | */ | |
| 194 | String saveOffCurrentDataToReadyFolder() throws IOException { | |
| 195 |
1
1. saveOffCurrentDataToReadyFolder : removed call to com/renomad/minum/database/DatabaseAppender::flush → KILLED |
flush(); |
| 196 | String newFileName = moveToReadyFolder(); | |
| 197 |
1
1. saveOffCurrentDataToReadyFolder : removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED |
createNewAppendFile(); |
| 198 |
1
1. saveOffCurrentDataToReadyFolder : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffCurrentDataToReadyFolder → KILLED |
return newFileName; |
| 199 | } | |
| 200 | ||
| 201 | /** | |
| 202 | * When we are done filling a file, move it to the ready | |
| 203 | * folder named by the date + time + millis. | |
| 204 | * @return the name of the new file | |
| 205 | */ | |
| 206 | private String moveToReadyFolder() throws IOException { | |
| 207 | String appendFile = simpleDateFormat.format(new java.util.Date()); | |
| 208 | Files.move(persistenceDirectory.resolve("currentAppendLog"), this.appendLogDirectory.resolve(appendFile)); | |
| 209 |
1
1. moveToReadyFolder : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::moveToReadyFolder → KILLED |
return appendFile; |
| 210 | } | |
| 211 | ||
| 212 | void flush() { | |
| 213 |
1
1. flush : removed call to com/renomad/minum/database/DatabaseAppender::flush → KILLED |
flush(this.bufferedWriter, this.logger); |
| 214 | this.bufferedWriterHasUnwrittenData = false; | |
| 215 | } | |
| 216 | ||
| 217 | static void flush(Writer writer, ILogger logger) { | |
| 218 | try { | |
| 219 |
1
1. flush : removed call to java/io/Writer::flush → KILLED |
writer.flush(); |
| 220 | } catch (IOException e) { | |
| 221 | logger.logAsyncError(() -> "Error while flushing in TimedFlusher: " + StacktraceUtils.stackTraceToString(e)); | |
| 222 | throw new DbException(e); | |
| 223 | } | |
| 224 | } | |
| 225 | } | |
Mutations | ||
| 94 |
1.1 |
|
| 96 |
1.1 |
|
| 111 |
1.1 |
|
| 130 |
1.1 2.2 |
|
| 131 |
1.1 |
|
| 135 |
1.1 |
|
| 140 |
1.1 |
|
| 141 |
1.1 |
|
| 142 |
1.1 2.2 |
|
| 143 |
1.1 |
|
| 148 |
1.1 |
|
| 149 |
1.1 |
|
| 163 |
1.1 |
|
| 164 |
1.1 |
|
| 183 |
1.1 2.2 |
|
| 184 |
1.1 |
|
| 195 |
1.1 |
|
| 197 |
1.1 |
|
| 198 |
1.1 |
|
| 209 |
1.1 |
|
| 213 |
1.1 |
|
| 219 |
1.1 |