Package org.voltdb.utils

Source Code of org.voltdb.utils.SnapshotConverter

/* This file is part of VoltDB.
* Copyright (C) 2008-2014 VoltDB Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
*/

package org.voltdb.utils;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.TreeSet;

import org.voltcore.logging.VoltLogger;
import org.voltdb.sysprocs.saverestore.SnapshotUtil;
import org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot;
import org.voltdb.sysprocs.saverestore.SnapshotUtil.SpecificSnapshotFilter;
import org.voltdb.sysprocs.saverestore.SnapshotUtil.TableFiles;

public class SnapshotConverter {
    private static final VoltLogger CONSOLE_LOG = new VoltLogger("CONSOLE");

    /**
     * @param args
     */
    public static void main(String[] args) {
        String snapshotName = null;
        ArrayList<File> directories = new ArrayList<File>();
        ArrayList<String> tables = new ArrayList<String>();
        File outdir = null;
        String type = null;
        char delimiter = '\0';

        for (int ii = 0; ii < args.length; ii++) {
            String arg = args[ii];
            if (arg.equals("--help")) {
                printHelpAndQuit(0);
            } else if (arg.equals("--dir")) {
                if (args.length < ii + 1) {
                    System.err.println("Error: Not enough args following --dirs");
                    printHelpAndQuit(-1);
                }
                boolean invalidDir = false;
                String dir = args[ii + 1];
                ii++;
                File f = new File(dir);
                if (!f.exists()) {
                    System.err.println("Error: " + dir + " does not exist");
                    invalidDir = true;
                }
                if (!f.canRead()) {
                    System.err.println("Error: " + dir + " does not have read permission set");
                    invalidDir = true;
                }
                if (!f.canExecute()) {
                    System.err.println("Error: " + dir + " does not have execute permission set");
                    invalidDir = true;
                }
                directories.add(f);
                if (invalidDir) {
                    System.exit(-1);
                }
            } else if (arg.equals("--timezone")) {
                if (args.length < ii + 1) {
                    System.err.println("Error: Not enough args following --timezone");
                    printHelpAndQuit(-1);
                }
                String tzId = args[ii + 1];
                ii++;
                VoltTableUtil.tz = TimeZone.getTimeZone(tzId);
            } else if (arg.equals("--table")) {
                if (args.length < ii + 1) {
                    System.err.println("Error: Not enough args following --tables");
                    printHelpAndQuit(-1);
                }
                tables.add(args[ii +1].toUpperCase());
                ii++;
            } else if (arg.equals("--outdir")) {
                if (args.length < ii + 1) {
                    System.err.println("Error: Not enough args following --outdir");
                    printHelpAndQuit(-1);
                }
                boolean invalidDir = false;
                outdir = new File(args[ii + 1]);
                if (!outdir.exists()) {
                    System.err.println("Error: " + outdir.getPath() + " does not exist");
                    invalidDir = true;
                }
                if (!outdir.canRead()) {
                    System.err.println("Error: " + outdir.getPath() + " does not have read permission set");
                    invalidDir = true;
                }
                if (!outdir.canExecute()) {
                    System.err.println("Error: " + outdir.getPath() + " does not have execute permission set");
                    invalidDir = true;
                }
                if (!outdir.canWrite()) {
                    System.err.println("Error: " + outdir.getPath() + " does not have write permission set");
                    invalidDir = true;
                }
                if (invalidDir) {
                    System.exit(-1);
                }
                ii++;
            else if (arg.equals("--type")) {
                if (args.length < ii + 1) {
                    System.err.println("Error: Not enough args following --type");
                    printHelpAndQuit(-1);
                }
                type = args[ii + 1];
                if (type.equalsIgnoreCase("csv")) {
                    delimiter = ',';
                } else if (type.equalsIgnoreCase("tsv")) {
                    delimiter = '\t';
                } else {
                    System.err.println("Error: --type must be one of CSV or TSV");
                    printHelpAndQuit(-1);
                }
                ii++;
            } else {
                if (snapshotName != null) {
                    System.err.println("Error: Multiple snapshots specified for conversion. First - " + snapshotName + " second " + args[ii]);
                    printHelpAndQuit(-1);
                }
                snapshotName = args[ii];
            }
        }
        boolean fail = false;
        if (snapshotName == null) {
            System.err.println("Error: No --name specified");
            fail = true;
        }
        if (directories.isEmpty()) {
            directories.add(new File("."));
        }
        if (tables.isEmpty()) {
            System.err.println("Error: No --tables specified");
            fail = true;
        }
        if (outdir == null) {
            outdir = new File(".");
        }
        if (type == null) {
            System.err.println("Error: No --type specified");
            fail = true;
        }

        if (fail) {
            printHelpAndQuit(-1);
        }

        Map<String, Snapshot> snapshots = new TreeMap<String, Snapshot>();
        HashSet<String> snapshotNames = new HashSet<String>();
        snapshotNames.add(snapshotName);
        SpecificSnapshotFilter filter = new SpecificSnapshotFilter(snapshotNames);
        for (File directory : directories) {
            SnapshotUtil.retrieveSnapshotFiles( directory, snapshots, filter, false, CONSOLE_LOG);
        }

        if (snapshots.size() > 1) {
            System.err.println("Error: Found " + snapshots.size() + " snapshots with specified name");
            int ii = 0;
            for (Snapshot entry : snapshots.values()) {
                System.err.println("Snapshot " + ii + " taken " + new Date(entry.getInstanceId().getTimestamp()));
                System.err.println("Files: ");
                for (File digest : entry.m_digests) {
                    System.err.println("\t" + digest.getPath());
                }
                for (Map.Entry<String, TableFiles> e2 : entry.m_tableFiles.entrySet()) {
                    System.err.println("\t" + e2.getKey());
                    for (File tableFile : e2.getValue().m_files) {
                        System.err.println("\t\t" + tableFile.getPath());
                    }
                }
                ii++;
            }
            System.exit(-1);
        }

        if (snapshots.size() < 1) {
            System.err.println("Error: Did not find any snapshots with the specified name");
            System.exit(-1);
        }

        /*
         * Build a plan for what partitions to pull from which save file
         */
        final Snapshot snapshot = snapshots.values().iterator().next();
        Map<String, Map<File, Set<Integer>>> tableToFilesWithPartitions =
            new TreeMap<String, Map<File, Set<Integer>>>();
        for (String tableName : tables) {
            if (!snapshot.m_tableFiles.containsKey(tableName)) {
                System.err.println("Error: Snapshot does not contain table " + tableName);
                System.exit(-1);
            }
            TableFiles tableFiles = snapshot.m_tableFiles.get(tableName);

            if (!tableFiles.m_isReplicated) {
                TreeSet<Integer> partitionsIds = new TreeSet<Integer>();
                Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
                for (int ii = 0; ii < tableFiles.m_files.size(); ii++) {
                    Set<Integer> validParititions = tableFiles.m_validPartitionIds.get(ii);
                    TreeSet<Integer> partitionsToTake = new TreeSet<Integer>(validParititions);
                    partitionsToTake.removeAll(partitionsIds);
                    partitionsIds.addAll(validParititions);
                    if (!partitionsToTake.isEmpty()) {
                        partitionsFromFile.put(tableFiles.m_files.get(ii), partitionsToTake);
                    }
                }
                int totalPartitionCount = tableFiles.m_totalPartitionCounts.get(0);
                if (!((partitionsIds.size() == totalPartitionCount) &&
                        (partitionsIds.first() == 0) &&
                        (partitionsIds.last() == totalPartitionCount - 1))) {
                    System.err.println("Error: Not all partitions present for table " + tableName);
                    fail = true;
                } else {
                    tableToFilesWithPartitions.put(tableName, partitionsFromFile);
                }
            } else {
                Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
                partitionsFromFile.put(tableFiles.m_files.get(0), null);
                tableToFilesWithPartitions.put(tableName, partitionsFromFile);
            }
        }

        if (fail) {
            System.exit(-1);
        }

        /*
         * For every output file that will be created attempt to instantiate and print an error
         * if the file already exists or couldn't be created.
         */
        for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
            String tableName = entry.getKey();
            File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());
            try {
                if (!outfile.createNewFile()) {
                    System.err.println("Error: Failed to create output file "
                            + outfile.getPath() + " for table " + tableName + "\n File already exists");
                    fail = true;
                }
            } catch (IOException e) {
                System.err.println(e.getMessage());
                System.err.println("Error: Failed to create output file "
                        + outfile.getPath() + " for table " + tableName);
                fail = true;
            }
        }

        if (fail) {
            System.exit(-1);
        }

        /*
         * Actually convert the tables and write the data to the appropriate destination
         */
        for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
            String tableName = entry.getKey();
            File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());

            Map<File, Set<Integer>> partitionsFromFile = entry.getValue();
            for (Map.Entry<File, Set<Integer>> e2 : partitionsFromFile.entrySet()) {
                File infile = e2.getKey();
                Set<Integer> partitionSet = e2.getValue();
                Integer partitions[] = null;
                if (partitionSet != null) {
                    partitions = new Integer[partitionSet.size()];
                    int ii = 0;
                    for (Integer partition : partitionSet) {
                        partitions[ii++] = partition;
                    }
                }
                try {
                    CSVTableSaveFile.convertTableSaveFile(delimiter, partitions, outfile, infile);
                } catch (Exception e) {
                    System.err.println(e.getMessage());
                    System.err.println("Error: Failed to convert " + infile.getPath() + " to " + outfile.getPath());
                }
            }
        }

        if (fail) {
            System.exit(-1);
        }
    }

    private static void printHelpAndQuit( int code) {
        System.out.println("java -cp <classpath> -Djava.library.path=<library path> org.voltdb.utils.SnapshotConverter --help");
        System.out.println("java -cp <classpath> -Djava.library.path=<library path> org.voltdb.utils.SnapshotConverter --dir dir1 --dir dir2 --dir dir3" +
                "--table table1 --table table2 --table table3 --type CSV|TSV --outdir dir snapshot_name --timezone GMT+0");
        System.exit(code);
    }
}
TOP

Related Classes of org.voltdb.utils.SnapshotConverter

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.