DatabaseAppender.java

1
package com.renomad.minum.database;
2
3
4
import com.renomad.minum.logging.ILogger;
5
import com.renomad.minum.state.Constants;
6
import com.renomad.minum.state.Context;
7
import com.renomad.minum.utils.IFileUtils;
8
import com.renomad.minum.utils.MyThread;
9
import com.renomad.minum.utils.ThrowingRunnable;
10
11
import java.io.BufferedWriter;
12
import java.io.IOException;
13
import java.io.Writer;
14
import java.nio.charset.StandardCharsets;
15
import java.nio.file.Path;
16
import java.nio.file.StandardOpenOption;
17
import java.time.Instant;
18
import java.time.ZoneId;
19
import java.util.List;
20
import java.util.concurrent.ExecutorService;
21
import java.util.concurrent.locks.ReentrantLock;
22
23
import static com.renomad.minum.database.AbstractDb.DATE_TIME_FORMATTER;
24
25
/**
26
 * This class provide the capability of appending database changes
27
 * to the disk, quickly and efficiently.
28
 */
29
final class DatabaseAppender {
30
31
    private final Path persistenceDirectory;
32
    private final IFileUtils fileUtils;
33
34
    Writer bufferedWriter;
35
36
    /**
37
     * if true, there is data in the buffered writer that needs to be
38
     * written to disk using {@link BufferedWriter#flush()}
39
     */
40
    private boolean bufferedWriterHasUnwrittenData;
41
42
    /**
43
     * This class field tracks the status of the loop which runs
44
     * a flush every second
45
     */
46
    private boolean flushLoopRunning;
47
48
    /**
49
     * The directory for this database
50
     */
51
    private final Path appendLogDirectory;
52
53
    /**
54
     * Used to create a thread that contains an inner loop
55
     * to flush the data to disk on a periodic basis
56
     */
57
    private final ExecutorService executorService;
58
59
    private final ILogger logger;
60
61
    private final ReentrantLock moveFileLock;
62
63
    /**
64
     * The maximum number of data's we will add to the append-only
65
     * file before we move on to a new file.
66
     */
67
    int maxAppendCount;
68
69
    /**
70
     * this is the current count of how many appends have
71
     * been made to the current database file.  Once it
72
     * exceeds a certain maximum, we'll switch to a
73
     * different file.
74
     */
75
    int appendCount;
76
77
    /**
78
     * This is the count of bytes that have been appended
79
     */
80
    private int appendBytes;
81
82
    DatabaseAppender(Path persistenceDirectory, Context context, IFileUtils fileUtils) throws IOException {
83
        this.fileUtils = fileUtils;
84
        this.persistenceDirectory = persistenceDirectory;
85
        this.appendLogDirectory = persistenceDirectory.resolve("append_logs");
86
        this.executorService = context.getExecutorService();
87
        this.logger = context.getLogger();
88
        Constants constants = context.getConstants();
89
        this.maxAppendCount = constants.maxAppendCount;
90 1 1. <init> : removed call to com/renomad/minum/utils/IFileUtils::makeDirectory → KILLED
        fileUtils.makeDirectory(this.appendLogDirectory);
91
        moveFileLock = new ReentrantLock();
92 1 1. <init> : removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED
        createNewAppendFile();
93
    }
94
95
    /**
96
     * Creates a new append-file (a file used for appending data) and
97
     * resets the append count to zero.
98
     */
99
    private void createNewAppendFile() throws IOException {
100
        Path currentAppendFile = this.persistenceDirectory.resolve("currentAppendLog");
101
102
        // if we are starting up with an existing currentAppendLog, set the appendCount
103
        // appropriately.  Otherwise, initialize to 0.  The currentAppendLog file is
104
        // never very large - it's mostly a temporary place to store incoming data
105
        // until we can store it off elsewhere. For that reason, it's not a performance
106
        // concern to read all the existing lines, just to get the count of current lines.
107 1 1. createNewAppendFile : negated conditional → KILLED
        if (fileUtils.exists(currentAppendFile)) {
108
            List<String> lines = fileUtils.readAllLines(currentAppendFile);
109
            appendCount = lines.size();
110
        } else {
111
            // reset the count to zero, we're starting a new file.
112
            logger.logDebug(() -> "Creating a new database append file. Previous file: %,d lines, %.2f megabytes".formatted(appendCount, ((double) appendBytes / 1_048_576.0)));
113
            appendCount = 0;
114
            appendBytes = 0;
115
        }
116
117 1 1. createNewAppendFile : negated conditional → KILLED
        if (bufferedWriter != null) {
118
            // Close the writer to avoid file descriptor leaks
119 1 1. createNewAppendFile : removed call to java/io/Writer::close → TIMED_OUT
            bufferedWriter.close();
120
        }
121
        bufferedWriter = fileUtils.newBufferedWriter(currentAppendFile, StandardCharsets.US_ASCII, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
122
    }
123
124
    /**
125
     * Appends new data to the end of a file.
126
     * @return if we created a new append file, we'll return the name of it. Otherwise, an empty string.
127
     */
128
    String appendToDatabase(DatabaseChangeAction action, String serializedData) throws IOException {
129
        String newlyCreatedFileName = "";
130 2 1. appendToDatabase : changed conditional boundary → KILLED
2. appendToDatabase : negated conditional → KILLED
        if (appendCount >= maxAppendCount) {
131 1 1. appendToDatabase : removed call to java/util/concurrent/locks/ReentrantLock::lock → KILLED
            moveFileLock.lock(); // block threads here if multiple are trying to get in - only one gets in at a time
132
            try {
133
                newlyCreatedFileName = saveOffWrapped(appendCount, maxAppendCount);
134
            } finally {
135 1 1. appendToDatabase : removed call to java/util/concurrent/locks/ReentrantLock::unlock → TIMED_OUT
                moveFileLock.unlock();
136
            }
137
        }
138
139
        bufferedWriter.append(action.toString()).append(' ').append(serializedData).append('\n');
140 1 1. appendToDatabase : removed call to com/renomad/minum/database/DatabaseAppender::setBufferedWriterHasUnwrittenData → KILLED
        setBufferedWriterHasUnwrittenData();
141 1 1. appendToDatabase : Replaced integer addition with subtraction → KILLED
        appendCount += 1;
142 2 1. appendToDatabase : Replaced integer addition with subtraction → TIMED_OUT
2. appendToDatabase : Replaced integer addition with subtraction → KILLED
        appendBytes += serializedData.length() + 8; // 8 includes the action (e.g. UPDATE), a space character, and a newline
143 1 1. appendToDatabase : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::appendToDatabase → KILLED
        return newlyCreatedFileName;
144
    }
145
146
    private void setBufferedWriterHasUnwrittenData() {
147
        bufferedWriterHasUnwrittenData = true;
148 1 1. setBufferedWriterHasUnwrittenData : negated conditional → KILLED
        if (!flushLoopRunning) {
149 1 1. setBufferedWriterHasUnwrittenData : removed call to com/renomad/minum/database/DatabaseAppender::initializeTimedFlusher → TIMED_OUT
            initializeTimedFlusher();
150
        }
151
    }
152
153
    /**
154
     * This method is kicked off when there is new data added to
155
     * the {@link BufferedWriter}.  While there is data to write, it
156
     * will wake up every second to flush the data.  Once there is
157
     * no more data, it will end.
158
     */
159
    private void initializeTimedFlusher() {
160
        ThrowingRunnable timedFlusherLoop = () -> {
161
            flushLoopRunning = true;
162
            Thread.currentThread().setName("database_timed_flusher");
163 1 1. lambda$initializeTimedFlusher$1 : negated conditional → KILLED
            while (bufferedWriterHasUnwrittenData) {
164 1 1. lambda$initializeTimedFlusher$1 : removed call to com/renomad/minum/database/DatabaseAppender::flush → KILLED
                flush();
165
166
                // this code only runs when there is data to add, so no need to take a
167
                // lot of waiting time.  But, if the data is coming fast and furious,
168
                // at least a small wait will allow greater efficiency.
169
                MyThread.sleep(50L);
170
            }
171
            flushLoopRunning = false;
172
        };
173
        executorService.submit(ThrowingRunnable.throwingRunnableWrapper(timedFlusherLoop, logger));
174
    }
175
176
    /**
177
     * This helper just wraps a method to enable easier testing.
178
     * @return the name of the newly-created file if the appendCount is greater or equal to maxAppendCount,
179
     * meaning that we moved on to calling {@link #saveOffCurrentDataToReadyFolder()},
180
     * empty-string otherwise.
181
     */
182
    String saveOffWrapped(int appendCount, int maxAppendCount) throws IOException {
183 2 1. saveOffWrapped : changed conditional boundary → KILLED
2. saveOffWrapped : negated conditional → KILLED
        if (appendCount >= maxAppendCount) {
184 1 1. saveOffWrapped : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffWrapped → KILLED
            return saveOffCurrentDataToReadyFolder();
185
        }
186
        return "";
187
    }
188
189
    /**
190
     * Move the append-only file to a new place to prepare for
191
     * consolidation, and reset the append count.
192
     * @return the name of the newly-created file
193
     */
194
    String saveOffCurrentDataToReadyFolder() throws IOException {
195 1 1. saveOffCurrentDataToReadyFolder : removed call to com/renomad/minum/database/DatabaseAppender::flush → TIMED_OUT
        flush();
196
        String newFileName = moveToReadyFolder(fileUtils, persistenceDirectory, appendLogDirectory);
197 1 1. saveOffCurrentDataToReadyFolder : removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED
        createNewAppendFile();
198 1 1. saveOffCurrentDataToReadyFolder : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffCurrentDataToReadyFolder → KILLED
        return newFileName;
199
    }
200
201
    /**
202
     * When we are done filling a file, move it to the ready
203
     * folder named by the date + time + millis.
204
     * @return the name of the new file
205
     */
206
    static String moveToReadyFolder(IFileUtils fileUtils, Path persistenceDirectory, Path appendLogDirectory) throws IOException {
207
        String appendFile = DATE_TIME_FORMATTER.format(Instant.now().atZone(ZoneId.of("UTC")));
208 1 1. moveToReadyFolder : removed call to com/renomad/minum/utils/IFileUtils::move → KILLED
        fileUtils.move(persistenceDirectory.resolve("currentAppendLog"), appendLogDirectory.resolve(appendFile));
209 1 1. moveToReadyFolder : replaced return value with "" for com/renomad/minum/database/DatabaseAppender::moveToReadyFolder → KILLED
        return appendFile;
210
    }
211
212
    void flush() throws IOException {
213 1 1. flush : removed call to java/io/Writer::flush → TIMED_OUT
        this.bufferedWriter.flush();
214
        this.bufferedWriterHasUnwrittenData = false;
215
    }
216
}

Mutations

90

1.1
Location : <init>
Killed by : com.renomad.minum.database.DatabaseAppenderTests
removed call to com/renomad/minum/utils/IFileUtils::makeDirectory → KILLED

92

1.1
Location : <init>
Killed by : com.renomad.minum.database.DatabaseAppenderTests
removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED

107

1.1
Location : createNewAppendFile
Killed by : com.renomad.minum.database.DatabaseAppenderTests
negated conditional → KILLED

117

1.1
Location : createNewAppendFile
Killed by : com.renomad.minum.database.DatabaseAppenderTests
negated conditional → KILLED

119

1.1
Location : createNewAppendFile
Killed by : none
removed call to java/io/Writer::close → TIMED_OUT

130

1.1
Location : appendToDatabase
Killed by : com.renomad.minum.database.DatabaseAppenderTests
changed conditional boundary → KILLED

2.2
Location : appendToDatabase
Killed by : com.renomad.minum.database.DatabaseAppenderTests
negated conditional → KILLED

131

1.1
Location : appendToDatabase
Killed by : com.renomad.minum.database.DatabaseAppenderTests
removed call to java/util/concurrent/locks/ReentrantLock::lock → KILLED

135

1.1
Location : appendToDatabase
Killed by : none
removed call to java/util/concurrent/locks/ReentrantLock::unlock → TIMED_OUT

140

1.1
Location : appendToDatabase
Killed by : com.renomad.minum.web.WebTests
removed call to com/renomad/minum/database/DatabaseAppender::setBufferedWriterHasUnwrittenData → KILLED

141

1.1
Location : appendToDatabase
Killed by : com.renomad.minum.web.WebTests
Replaced integer addition with subtraction → KILLED

142

1.1
Location : appendToDatabase
Killed by : none
Replaced integer addition with subtraction → TIMED_OUT

2.2
Location : appendToDatabase
Killed by : com.renomad.minum.web.WebTests
Replaced integer addition with subtraction → KILLED

143

1.1
Location : appendToDatabase
Killed by : com.renomad.minum.database.DatabaseAppenderTests
replaced return value with "" for com/renomad/minum/database/DatabaseAppender::appendToDatabase → KILLED

148

1.1
Location : setBufferedWriterHasUnwrittenData
Killed by : com.renomad.minum.web.WebTests
negated conditional → KILLED

149

1.1
Location : setBufferedWriterHasUnwrittenData
Killed by : none
removed call to com/renomad/minum/database/DatabaseAppender::initializeTimedFlusher → TIMED_OUT

163

1.1
Location : lambda$initializeTimedFlusher$1
Killed by : com.renomad.minum.web.WebTests
negated conditional → KILLED

164

1.1
Location : lambda$initializeTimedFlusher$1
Killed by : com.renomad.minum.web.WebTests
removed call to com/renomad/minum/database/DatabaseAppender::flush → KILLED

183

1.1
Location : saveOffWrapped
Killed by : com.renomad.minum.database.DatabaseAppenderTests
changed conditional boundary → KILLED

2.2
Location : saveOffWrapped
Killed by : com.renomad.minum.database.DatabaseAppenderTests
negated conditional → KILLED

184

1.1
Location : saveOffWrapped
Killed by : com.renomad.minum.database.DatabaseAppenderTests
replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffWrapped → KILLED

195

1.1
Location : saveOffCurrentDataToReadyFolder
Killed by : none
removed call to com/renomad/minum/database/DatabaseAppender::flush → TIMED_OUT

197

1.1
Location : saveOffCurrentDataToReadyFolder
Killed by : com.renomad.minum.database.DatabaseAppenderTests
removed call to com/renomad/minum/database/DatabaseAppender::createNewAppendFile → KILLED

198

1.1
Location : saveOffCurrentDataToReadyFolder
Killed by : com.renomad.minum.database.DatabaseAppenderTests
replaced return value with "" for com/renomad/minum/database/DatabaseAppender::saveOffCurrentDataToReadyFolder → KILLED

208

1.1
Location : moveToReadyFolder
Killed by : com.renomad.minum.database.DatabaseAppenderTests
removed call to com/renomad/minum/utils/IFileUtils::move → KILLED

209

1.1
Location : moveToReadyFolder
Killed by : com.renomad.minum.database.DatabaseAppenderTests
replaced return value with "" for com/renomad/minum/database/DatabaseAppender::moveToReadyFolder → KILLED

213

1.1
Location : flush
Killed by : none
removed call to java/io/Writer::flush → TIMED_OUT

Active mutators

Tests examined


Report generated by PIT 1.17.0