DatabaseConsolidator.java

package com.renomad.minum.database;

import com.renomad.minum.logging.ILogger;

import com.renomad.minum.state.Context;

import com.renomad.minum.utils.IFileUtils;

import java.io.IOException;

import java.nio.charset.StandardCharsets;

import java.nio.file.Path;

import java.text.ParseException;

import java.util.*;

import java.util.stream.Collectors;

import static com.renomad.minum.database.ChecksumUtility.buildChecksum;

import static com.renomad.minum.database.ChecksumUtility.compareWithChecksum;

import static com.renomad.minum.database.DatabaseAppender.simpleDateFormat;

/**

 * Consolidates the database append logs.

 * <br>

 * As the append logs get filled up, the consolidator comes

 * along after to analyze those changes and determine a

 * consolidated version.  For example, if the append logs

 * have three updates for a particular element, then the consolidated file

 * will have just the last update.

 */

final class DatabaseConsolidator {

    /**

     * This is the path to the append-only files, where incoming

     * changes to the data are quickly stored.

     */

    private final Path appendLogDirectory;

    /**

     * This is the path to where we store consolidated data, so that

     * database startup is as fast as possible.

     */

    private final Path consolidatedDataDirectory;

    private final ILogger logger;

    private final int maxLinesPerFile;

    private final IFileUtils fileUtils;

    /**

     * This represents an instruction for how to change the overall consolidated

     * database files on disk.  Instructions are either to UPDATE or DELETE. This

     * also encapsulates the data we're updating.

     */

    /**

     * Loop through all the append-only files

     */

    void consolidate() throws IOException, ParseException {

        } else {

        }

        // process the files in order.  This does potentially cause

        // multiple updates for the consolidated files, but that's

        // safer than building up too large a structure in memory

        // before writing, and in any case, we're prioritizing efficiency

        // so there should only be one write to each file per loop.

        //

        // after each append-only file is fully processed, it gets deleted.

    /**

     * The expectation is that after we finish reading the X lines in

     * this append log, we will have a set of clear instructions to

     * apply to our previously consolidated files. There should end up

     * being just one action for each id - update or delete.

     * <br>

     * Build a data structure holding instructions for the next step.

     */

    static void processAppendLogFile(String filename, Path appendLogDirectory, IFileUtils fileUtils,

                                     int maxLinesPerFile, ILogger logger, Path consolidatedDataDirectory) throws IOException {

        // process each line from the file

            // the trick here is that by using a Map, only the last item added will remain at the end

        // now we have the concise list of state changes, but the next step is figuring out how

        // to organize them by their destination.  consolidated files will be grouped somehow.

        // For example, indexes 1 - 1000, 1001-2000, etc (there may be more than 1000 per file).

        // <br>

        // So, we will group our data that way,

        // and then efficiently update the files (a bad outcome, in contrast, would be updating

        // the files multiple times each).

        // delete the file

    /**

     * Given a {@link Map} of database change instructions, grouped by keys representing the

     * first index in a group of indexes (like 1 to 100, or 101 to 200, etc), write the

     * data to files, with consideration for what might already exist.  That is to say,

     * if we are adding grouped instructions to an existing file such as "1_to_100", then

     * we want to merge our incoming data with what is already there.  Otherwise, we are just

     * creating a new file.

     */

    static void rewriteFiles(Map<Long, Collection<DatabaseChangeInstruction>> groupedInstructions,

                             IFileUtils fileUtils, int maxLinesPerFile, ILogger logger, Path consolidatedDataDirectory) throws IOException {

            List<String> data;

            // if the file doesn't exist, we'll just start with an empty list. If it

            // does exist, read its lines into a List data structure.

            } else {

            }

            // update the data in memory per the instructions

            // write the data to disk

            // write a hash of the data to use as a checksum.  This value will be checked

            // when reading the data later on, to confirm nothing has changed since writing.

    /**

     * Reads data from consolidated file, confirming the checksum in the process.

     */

    static List<String> readConsolidatedFileWithChecksum(Path fullPathToConsolidatedFile, IFileUtils fileUtils) throws IOException {

        // get all the data from the consolidated file

    }

    /**

     * Here, we have raw lines of data from a file, and a list of instructions for updating

     * that data.  We will organize the raw data better, apply the instructions, and return

     * the updated data

     *

     * @param linesOfData  raw lines of data from a file

     * @param instructions details of how to change the data in the file, either UPDATE or DELETE

     * @return an updated and sorted list of strings (sorted by index, which is the first value on each line)

     */

    static Collection<String> updateData(String filename, List<String> linesOfData, Collection<DatabaseChangeInstruction> instructions) {

        // put the original data into a map

            // the first pipe symbol is where the index number ends.  Apologies for

            // the overlap of terms here, index and index.

            }

            long dataIndexLong;

            try {

        // change that data per instructions

            } else {

                // only other option is DELETE

            }

    }

    /**

     * This method will group the instructions for changes to the database by which

     * consolidated files they apply to, so that we only need to make one change

     * to each file.  Files are named like this: 1, 1001, etc., or

     * in other words, the starting index of each set of consolidated data.

     * @param databaseChangeInstructionMap this is a map between keys representing the

     *                                     index of the data, and the data itself.

     * @return a map consisting of keys representing the target file for the data, and

     * a collection of DatabaseChangeInstruction data to place in that file.

     */

    static Map<Long, Collection<DatabaseChangeInstruction>> groupInstructionsByPartition(

            Map<Long, DatabaseChangeInstruction> databaseChangeInstructionMap, int maxLinesPerFile) {

        // initialize a data structure to store our results

        // loop through the incoming data, grouping and ordering as necessary

            // determine the expected filename for this file.  For example, if the index is 1234, then

            // the filename should be 1001

            // If there is no key found, we need to add one, and add a new collection

            // add a new item to the collection for this filename

    }

    /**

     * read first 6 characters - is it update or delete?

     * skip a character

     * read digits until we hit a pipe symbol, that's our index.

     * read the rest of the content

     */

    static DatabaseChangeInstruction parseDatabaseChangeInstructionString(String databaseInstructionString, String filename) {

        DatabaseChangeAction action;

        } else {

        }

        // confusing overlap of terms - index is used here to mean two things:

        // a) where we find the first pipe symbol

        // b) the index value of the data

        }

    }

    /**

     * Given a directory, convert the list of files into a sorted

     * list of dates.

     * @return a sorted list of dates, or an empty list if nothing found

     */

    static List<Date> getSortedAppendLogs(Path appendLogDirectory) throws ParseException {

        // get the list of file names, which are date-time stamps

        // if there aren't any append-only files, bail out with an empty list

        }

        // sort

    }

    /**

     * Convert a list of filenames to a list of dates

     */

    static List<Date> convertFileListToDateList(String[] listOfFiles) throws ParseException {

        // initialize a list which will hold the dates associated with each file name

        // convert the names to dates

        }

    }

}

Mutations