will/LanternPowerMonitor
1
0
Fork 0
mirror of https://github.com/zyphlar/LanternPowerMonitor.git synced 2024-03-08 14:07:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow exporting all data in bson, json, or csv formats.

Browse Source
This commit is contained in:
MarkBryanMilligan 2022-01-29 23:10:04 -06:00
parent eaf1e4504f
commit 94ebf5fa93
3 changed files with 45 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
currentmonitor/lantern-dataaccess-currentmonitor/src/main/java/com/lanternsoftware/dataaccess/currentmonitor/MongoCurrentMonitorDao.java
View File

@ -46,14 +46,10 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -68,7 +64,8 @@ import java.util.TimerTask;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class MongoCurrentMonitorDao implements CurrentMonitorDao { public class MongoCurrentMonitorDao implements CurrentMonitorDao {
private static final Logger logger = LoggerFactory.getLogger(MongoCurrentMonitorDao.class); private static final Logger logger = LoggerFactory.getLogger(MongoCurrentMonitorDao.class);
@ -143,8 +140,6 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao {
Date start = _month; Date start = _month;
Date end = DateUtils.getEndOfMonth(_month, tz); Date end = DateUtils.getEndOfMonth(_month, tz);
BreakerConfig config = getConfig(_accountId); //TODO: get historical config for archive month in case it's changed since then. BreakerConfig config = getConfig(_accountId); //TODO: get historical config for archive month in case it's changed since then.
List<Breaker> breakers = CollectionUtils.filter(config.getAllBreakers(), _b -> !NullUtils.isOneOf(_b.getType(), BreakerType.DOUBLE_POLE_BOTTOM, BreakerType.EMPTY));
breakers.sort(Comparator.comparing(Breaker::getPanel).thenComparing(Breaker::getSpace));
Map<Integer, Integer> breakerKeys = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey, _b -> Breaker.intKey(_b.getPanel(), _b.getType() == BreakerType.DOUBLE_POLE_BOTTOM ? _b.getSpace() - 2 : _b.getSpace())); Map<Integer, Integer> breakerKeys = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey, _b -> Breaker.intKey(_b.getPanel(), _b.getType() == BreakerType.DOUBLE_POLE_BOTTOM ? _b.getSpace() - 2 : _b.getSpace()));
Map<Integer, List<Float>> minuteReadings = new HashMap<>(); Map<Integer, List<Float>> minuteReadings = new HashMap<>();
MonthlyEnergyArchive archive = new MonthlyEnergyArchive(); MonthlyEnergyArchive archive = new MonthlyEnergyArchive();
@ -161,8 +156,6 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao {
HubPowerMinute m = null; HubPowerMinute m = null;
if (i.hasNext()) if (i.hasNext())
m = i.next(); m = i.next();
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
df.setTimeZone(TimeZone.getTimeZone("UTC"));
while (i.hasNext()) { while (i.hasNext()) {
if (m == null) if (m == null)
break; break;
@ -189,6 +182,14 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao {
if (m != null) if (m != null)
addReadings(minute, bytesInDay, minuteReadings, dayReadings); addReadings(minute, bytesInDay, minuteReadings, dayReadings);
List<BreakerEnergyArchive> breakerEnergies = new ArrayList<>(); List<BreakerEnergyArchive> breakerEnergies = new ArrayList<>();
byte[] nanArray = new byte[bytesInDay];
ByteBuffer nanBuffer = ByteBuffer.wrap(nanArray);
for (int offset = 0; offset < bytesInDay; offset += 4) {
nanBuffer.putFloat(offset, Float.NaN);
}
for (int key : breakerKeys.keySet()) {
dayReadings.computeIfAbsent(key, _k->nanArray);
}
for (Entry<Integer, byte[]> be : dayReadings.entrySet()) { for (Entry<Integer, byte[]> be : dayReadings.entrySet()) {
BreakerEnergyArchive breakerEnergy = new BreakerEnergyArchive(); BreakerEnergyArchive breakerEnergy = new BreakerEnergyArchive();
breakerEnergy.setPanel(Breaker.intKeyToPanel(be.getKey())); breakerEnergy.setPanel(Breaker.intKeyToPanel(be.getKey()));
@ -209,7 +210,7 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao {
t.stop(); t.stop();
DebugTimer t2 = new DebugTimer("Zip Archive and write to disk for account" + archive.getAccountId()); DebugTimer t2 = new DebugTimer("Zip Archive and write to disk for account" + archive.getAccountId());
OutputStream os = null; ZipOutputStream os = null;
try { try {
File partialPath = new File(LanternFiles.BACKUP_DEST_PATH + archive.getAccountId()+File.separator + "partial"); File partialPath = new File(LanternFiles.BACKUP_DEST_PATH + archive.getAccountId()+File.separator + "partial");
FileUtils.deleteDirectory(partialPath); FileUtils.deleteDirectory(partialPath);
@ -217,13 +218,18 @@ public class MongoCurrentMonitorDao implements CurrentMonitorDao {
String backupPath = LanternFiles.BACKUP_DEST_PATH + archive.getAccountId() + File.separator; String backupPath = LanternFiles.BACKUP_DEST_PATH + archive.getAccountId() + File.separator;
if (!archive.isComplete(tz)) if (!archive.isComplete(tz))
backupPath += "partial" + File.separator; backupPath += "partial" + File.separator;
os = new GZIPOutputStream(new FileOutputStream(backupPath + archive.getMonth().getTime() + ".zip")) {{def.setLevel(Deflater.BEST_SPEED);}}; os = new ZipOutputStream(new FileOutputStream(backupPath + archive.getMonth().getTime() + ".zip"));
os.setLevel(Deflater.BEST_SPEED);
ZipEntry e = new ZipEntry(DateUtils.format("MMMM-yyyy", tz, archive.getMonth()) + ".bson");
os.putNextEntry(e);
int batchSize = bson.length / 50; int batchSize = bson.length / 50;
for (int offset = 0; offset < bson.length; offset += batchSize) { for (int offset = 0; offset < bson.length; offset += batchSize) {
os.write(bson, offset, Math.min(batchSize, bson.length - offset)); os.write(bson, offset, Math.min(batchSize, bson.length - offset));
status.setProgress(50 + (50f * offset / bson.length)); status.setProgress(50 + (50f * offset / bson.length));
putArchiveStatus(status); putArchiveStatus(status);
} }
os.closeEntry();
os.flush();
} catch (Exception _e) { } catch (Exception _e) {
logger.error("Failed to write export file", _e); logger.error("Failed to write export file", _e);
} finally { } finally {

38
currentmonitor/lantern-service-currentmonitor/src/main/java/com/lanternsoftware/currentmonitor/servlet/console/ExportServlet.java
View File

@ -35,7 +35,9 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TimeZone; import java.util.TimeZone;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
@WebServlet("/export/*") @WebServlet("/export/*")
public class ExportServlet extends SecureConsoleServlet { public class ExportServlet extends SecureConsoleServlet {
@ -47,21 +49,27 @@ public class ExportServlet extends SecureConsoleServlet {
String[] path = path(_req); String[] path = path(_req);
if (path.length > 1) { if (path.length > 1) {
synchronized (this) { synchronized (this) {
InputStream is = Globals.dao.streamArchive(_authCode.getAccountId(), new Date(DaoSerializer.toLong(path[0]))); Date month = new Date(DaoSerializer.toLong(path[0]));
InputStream is = Globals.dao.streamArchive(_authCode.getAccountId(), month);
if (is == null) { if (is == null) {
redirect(_rep, _req.getContextPath() + "/export"); redirect(_rep, _req.getContextPath() + "/export");
return; return;
} }
OutputStream os = null; OutputStream os = null;
GZIPOutputStream gout = null; ZipInputStream zis = null;
ZipOutputStream zos = null;
JsonWriter jsonWriter = null; JsonWriter jsonWriter = null;
try { try {
os = _rep.getOutputStream(); os = _rep.getOutputStream();
if (NullUtils.makeNotNull(path[1]).contains("csv")) { if (NullUtils.makeNotNull(path[1]).contains("csv")) {
BreakerConfig config = Globals.dao.getConfig(_authCode.getAccountId()); //TODO: get historical config for this month in case it's changed since then. BreakerConfig config = Globals.dao.getConfig(_authCode.getAccountId()); //TODO: get historical config for this month in case it's changed since then.
Map<Integer, Breaker> breakers = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey); Map<Integer, Breaker> breakers = CollectionUtils.transformToMap(config.getAllBreakers(), Breaker::getIntKey);
os = new GZIPOutputStream(os) {{def.setLevel(Deflater.BEST_SPEED);}}; zos = new ZipOutputStream(os);
MonthlyEnergyArchive archive = DaoSerializer.fromZipBson(IOUtils.toByteArray(is), MonthlyEnergyArchive.class); zos.setLevel(Deflater.BEST_SPEED);
zos.putNextEntry(new ZipEntry(DateUtils.format("MMMM-yyyy", tz, month) + ".csv"));
zis = new ZipInputStream(is);
zis.getNextEntry();
MonthlyEnergyArchive archive = DaoSerializer.fromBson(IOUtils.toByteArray(zis), MonthlyEnergyArchive.class);
DailyEnergyArchive fday = CollectionUtils.getFirst(archive.getDays()); DailyEnergyArchive fday = CollectionUtils.getFirst(archive.getDays());
if (fday == null) { if (fday == null) {
redirect(_rep, _req.getContextPath() + "/export"); redirect(_rep, _req.getContextPath() + "/export");
@ -80,7 +88,7 @@ public class ExportServlet extends SecureConsoleServlet {
header.append("\n"); header.append("\n");
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
df.setTimeZone(TimeZone.getTimeZone("UTC")); df.setTimeZone(TimeZone.getTimeZone("UTC"));
os.write(NullUtils.toByteArray(header.toString())); zos.write(NullUtils.toByteArray(header.toString()));
Date dayStart = archive.getMonth(); Date dayStart = archive.getMonth();
for (DailyEnergyArchive day : CollectionUtils.makeNotNull(archive.getDays())) { for (DailyEnergyArchive day : CollectionUtils.makeNotNull(archive.getDays())) {
Date dayEnd = DateUtils.addDays(dayStart, 1, tz); Date dayEnd = DateUtils.addDays(dayStart, 1, tz);
@ -98,16 +106,23 @@ public class ExportServlet extends SecureConsoleServlet {
} }
} }
line.append("\n"); line.append("\n");
os.write(NullUtils.toByteArray(line.toString())); zos.write(NullUtils.toByteArray(line.toString()));
} }
dayStart = dayEnd;
} }
zos.flush();
return; return;
} }
if (NullUtils.makeNotNull(path[1]).contains("json")) { if (NullUtils.makeNotNull(path[1]).contains("json")) {
DaoEntity archive = DaoSerializer.fromZipBson(IOUtils.toByteArray(is)); zis = new ZipInputStream(is);
gout = new GZIPOutputStream(os) {{def.setLevel(Deflater.BEST_SPEED);}}; zis.getNextEntry();
jsonWriter = new JsonWriter(new OutputStreamWriter(gout, StandardCharsets.UTF_8), DaoSerializer.JSON_COMPACT_SETTINGS); DaoEntity archive = DaoSerializer.fromBson(IOUtils.toByteArray(zis));
zos = new ZipOutputStream(os);
zos.setLevel(Deflater.BEST_SPEED);
zos.putNextEntry(new ZipEntry(DateUtils.format("MMMM-yyyy", tz, month) + ".json"));
jsonWriter = new JsonWriter(new OutputStreamWriter(zos, StandardCharsets.UTF_8), DaoSerializer.JSON_PRETTY_SETTINGS);
new DocumentCodec().encode(jsonWriter, archive.toDocument(), EncoderContext.builder().build()); new DocumentCodec().encode(jsonWriter, archive.toDocument(), EncoderContext.builder().build());
jsonWriter.flush();
return; return;
} }
IOUtils.copy(is, os); IOUtils.copy(is, os);
@ -119,7 +134,8 @@ public class ExportServlet extends SecureConsoleServlet {
} finally { } finally {
IOUtils.closeQuietly(is); IOUtils.closeQuietly(is);
IOUtils.closeQuietly(jsonWriter); IOUtils.closeQuietly(jsonWriter);
IOUtils.closeQuietly(gout); IOUtils.closeQuietly(zos);
IOUtils.closeQuietly(zis);
IOUtils.closeQuietly(os); IOUtils.closeQuietly(os);
} }
} }

2
currentmonitor/lantern-service-currentmonitor/src/main/resources/templates/export.ftl
View File

@ -6,7 +6,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<#if inprogress><meta http-equiv="refresh" content="1"></#if> <#if inprogress><meta http-equiv="refresh" content="1"></#if>
<title>Lantern Console</title> <title>Lantern Console</title>
<link href="${link_prefix}/bootstrap/css/bootstrap.min.css" rel="stylesheet"> <link href="${link_prefix}bootstrap/css/bootstrap.min.css" rel="stylesheet">
<link href="${link_prefix}bootstrap/css/style.min.css" rel="stylesheet"> <link href="${link_prefix}bootstrap/css/style.min.css" rel="stylesheet">
</head> </head>
<body> <body>