diff --git a/currentmonitor/lantern-dataaccess-currentmonitor/src/main/java/com/lanternsoftware/dataaccess/currentmonitor/MongoCurrentMonitorDao.java b/currentmonitor/lantern-dataaccess-currentmonitor/src/main/java/com/lanternsoftware/dataaccess/currentmonitor/MongoCurrentMonitorDao.java index 779b8f1..d12a9c7 100644 --- a/currentmonitor/lantern-dataaccess-currentmonitor/src/main/java/com/lanternsoftware/dataaccess/currentmonitor/MongoCurrentMonitorDao.java +++ b/currentmonitor/lantern-dataaccess-currentmonitor/src/main/java/com/lanternsoftware/dataaccess/currentmonitor/MongoCurrentMonitorDao.java @@ -46,14 +46,10 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; -import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -68,7 +64,8 @@ import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.zip.Deflater; -import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; public class MongoCurrentMonitorDao implements CurrentMonitorDao { private static final Logger logger = LoggerFactory.getLogger(MongoCurrentMonitorDao.class); @@ -143,8 +140,6 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao { Date start = _month; Date end = DateUtils.getEndOfMonth(_month, tz); BreakerConfig config = getConfig(_accountId); //TODO: get historical config for archive month in case it's changed since then. - List breakers = CollectionUtils.filter(config.getAllBreakers(), _b -> !NullUtils.isOneOf(_b.getType(), BreakerType.DOUBLE_POLE_BOTTOM, BreakerType.EMPTY)); - breakers.sort(Comparator.comparing(Breaker::getPanel).thenComparing(Breaker::getSpace)); Map breakerKeys = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey, _b -> Breaker.intKey(_b.getPanel(), _b.getType() == BreakerType.DOUBLE_POLE_BOTTOM ? _b.getSpace() - 2 : _b.getSpace())); Map> minuteReadings = new HashMap<>(); MonthlyEnergyArchive archive = new MonthlyEnergyArchive(); @@ -161,8 +156,6 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao { HubPowerMinute m = null; if (i.hasNext()) m = i.next(); - DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - df.setTimeZone(TimeZone.getTimeZone("UTC")); while (i.hasNext()) { if (m == null) break; @@ -189,6 +182,14 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao { if (m != null) addReadings(minute, bytesInDay, minuteReadings, dayReadings); List breakerEnergies = new ArrayList<>(); + byte[] nanArray = new byte[bytesInDay]; + ByteBuffer nanBuffer = ByteBuffer.wrap(nanArray); + for (int offset = 0; offset < bytesInDay; offset += 4) { + nanBuffer.putFloat(offset, Float.NaN); + } + for (int key : breakerKeys.keySet()) { + dayReadings.computeIfAbsent(key, _k->nanArray); + } for (Entry be : dayReadings.entrySet()) { BreakerEnergyArchive breakerEnergy = new BreakerEnergyArchive(); breakerEnergy.setPanel(Breaker.intKeyToPanel(be.getKey())); @@ -209,7 +210,7 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao { t.stop(); DebugTimer t2 = new DebugTimer("Zip Archive and write to disk for account" + archive.getAccountId()); - OutputStream os = null; + ZipOutputStream os = null; try { File partialPath = new File(LanternFiles.BACKUP_DEST_PATH + archive.getAccountId()+File.separator + "partial"); FileUtils.deleteDirectory(partialPath); @@ -217,13 +218,18 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao { String backupPath = LanternFiles.BACKUP_DEST_PATH + archive.getAccountId() + File.separator; if (!archive.isComplete(tz)) backupPath += "partial" + File.separator; - os = new GZIPOutputStream(new FileOutputStream(backupPath + archive.getMonth().getTime() + ".zip")) {{def.setLevel(Deflater.BEST_SPEED);}}; + os = new ZipOutputStream(new FileOutputStream(backupPath + archive.getMonth().getTime() + ".zip")); + os.setLevel(Deflater.BEST_SPEED); + ZipEntry e = new ZipEntry(DateUtils.format("MMMM-yyyy", tz, archive.getMonth()) + ".bson"); + os.putNextEntry(e); int batchSize = bson.length / 50; for (int offset = 0; offset < bson.length; offset += batchSize) { os.write(bson, offset, Math.min(batchSize, bson.length - offset)); status.setProgress(50 + (50f * offset / bson.length)); putArchiveStatus(status); } + os.closeEntry(); + os.flush(); } catch (Exception _e) { logger.error("Failed to write export file", _e); } finally { diff --git a/currentmonitor/lantern-service-currentmonitor/src/main/java/com/lanternsoftware/currentmonitor/servlet/console/ExportServlet.java b/currentmonitor/lantern-service-currentmonitor/src/main/java/com/lanternsoftware/currentmonitor/servlet/console/ExportServlet.java index deec364..ff35712 100644 --- a/currentmonitor/lantern-service-currentmonitor/src/main/java/com/lanternsoftware/currentmonitor/servlet/console/ExportServlet.java +++ b/currentmonitor/lantern-service-currentmonitor/src/main/java/com/lanternsoftware/currentmonitor/servlet/console/ExportServlet.java @@ -35,7 +35,9 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.zip.Deflater; -import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; @WebServlet("/export/*") public class ExportServlet extends SecureConsoleServlet { @@ -47,21 +49,27 @@ public class ExportServlet extends SecureConsoleServlet { String[] path = path(_req); if (path.length > 1) { synchronized (this) { - InputStream is = Globals.dao.streamArchive(_authCode.getAccountId(), new Date(DaoSerializer.toLong(path[0]))); + Date month = new Date(DaoSerializer.toLong(path[0])); + InputStream is = Globals.dao.streamArchive(_authCode.getAccountId(), month); if (is == null) { redirect(_rep, _req.getContextPath() + "/export"); return; } OutputStream os = null; - GZIPOutputStream gout = null; + ZipInputStream zis = null; + ZipOutputStream zos = null; JsonWriter jsonWriter = null; try { os = _rep.getOutputStream(); if (NullUtils.makeNotNull(path[1]).contains("csv")) { BreakerConfig config = Globals.dao.getConfig(_authCode.getAccountId()); //TODO: get historical config for this month in case it's changed since then. Map breakers = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey); - os = new GZIPOutputStream(os) {{def.setLevel(Deflater.BEST_SPEED);}}; - MonthlyEnergyArchive archive = DaoSerializer.fromZipBson(IOUtils.toByteArray(is), MonthlyEnergyArchive.class); + zos = new ZipOutputStream(os); + zos.setLevel(Deflater.BEST_SPEED); + zos.putNextEntry(new ZipEntry(DateUtils.format("MMMM-yyyy", tz, month) + ".csv")); + zis = new ZipInputStream(is); + zis.getNextEntry(); + MonthlyEnergyArchive archive = DaoSerializer.fromBson(IOUtils.toByteArray(zis), MonthlyEnergyArchive.class); DailyEnergyArchive fday = CollectionUtils.getFirst(archive.getDays()); if (fday == null) { redirect(_rep, _req.getContextPath() + "/export"); @@ -80,7 +88,7 @@ public class ExportServlet extends SecureConsoleServlet { header.append("\n"); DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); df.setTimeZone(TimeZone.getTimeZone("UTC")); - os.write(NullUtils.toByteArray(header.toString())); + zos.write(NullUtils.toByteArray(header.toString())); Date dayStart = archive.getMonth(); for (DailyEnergyArchive day : CollectionUtils.makeNotNull(archive.getDays())) { Date dayEnd = DateUtils.addDays(dayStart, 1, tz); @@ -98,16 +106,23 @@ public class ExportServlet extends SecureConsoleServlet { } } line.append("\n"); - os.write(NullUtils.toByteArray(line.toString())); + zos.write(NullUtils.toByteArray(line.toString())); } + dayStart = dayEnd; } + zos.flush(); return; } if (NullUtils.makeNotNull(path[1]).contains("json")) { - DaoEntity archive = DaoSerializer.fromZipBson(IOUtils.toByteArray(is)); - gout = new GZIPOutputStream(os) {{def.setLevel(Deflater.BEST_SPEED);}}; - jsonWriter = new JsonWriter(new OutputStreamWriter(gout, StandardCharsets.UTF_8), DaoSerializer.JSON_COMPACT_SETTINGS); + zis = new ZipInputStream(is); + zis.getNextEntry(); + DaoEntity archive = DaoSerializer.fromBson(IOUtils.toByteArray(zis)); + zos = new ZipOutputStream(os); + zos.setLevel(Deflater.BEST_SPEED); + zos.putNextEntry(new ZipEntry(DateUtils.format("MMMM-yyyy", tz, month) + ".json")); + jsonWriter = new JsonWriter(new OutputStreamWriter(zos, StandardCharsets.UTF_8), DaoSerializer.JSON_PRETTY_SETTINGS); new DocumentCodec().encode(jsonWriter, archive.toDocument(), EncoderContext.builder().build()); + jsonWriter.flush(); return; } IOUtils.copy(is, os); @@ -119,7 +134,8 @@ public class ExportServlet extends SecureConsoleServlet { } finally { IOUtils.closeQuietly(is); IOUtils.closeQuietly(jsonWriter); - IOUtils.closeQuietly(gout); + IOUtils.closeQuietly(zos); + IOUtils.closeQuietly(zis); IOUtils.closeQuietly(os); } } diff --git a/currentmonitor/lantern-service-currentmonitor/src/main/resources/templates/export.ftl b/currentmonitor/lantern-service-currentmonitor/src/main/resources/templates/export.ftl index 0aaadf2..fa58307 100644 --- a/currentmonitor/lantern-service-currentmonitor/src/main/resources/templates/export.ftl +++ b/currentmonitor/lantern-service-currentmonitor/src/main/resources/templates/export.ftl @@ -6,7 +6,7 @@ <#if inprogress> Lantern Console - +