1 package org.lsst.ccs.localdb.statusdb.trendServer;
2
3 import com.sun.jersey.spi.resource.Singleton;
4 import com.sun.net.httpserver.HttpHandler;
5 import com.sun.net.httpserver.HttpServer;
6 import java.io.IOException;
7 import java.math.BigInteger;
8 import java.net.InetSocketAddress;
9 import java.util.ArrayList;
10 import java.util.HashMap;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.Map;
14 import java.util.Properties;
15 import java.util.Set;
16 import javax.naming.Context;
17 import javax.naming.InitialContext;
18 import javax.naming.NamingException;
19 import javax.ws.rs.DefaultValue;
20 import javax.ws.rs.GET;
21
22 import javax.ws.rs.Path;
23 import javax.ws.rs.PathParam;
24 import javax.ws.rs.QueryParam;
25 import javax.ws.rs.core.Application;
26 import javax.ws.rs.ext.RuntimeDelegate;
27 import javax.xml.bind.annotation.XmlElement;
28 import javax.xml.bind.annotation.XmlElementWrapper;
29 import javax.xml.bind.annotation.XmlRootElement;
30 import org.hibernate.Query;
31 import org.hibernate.SQLQuery;
32 import org.hibernate.Session;
33 import org.hibernate.SessionFactory;
34 import org.lsst.ccs.localdb.statusdb.model.DataDesc;
35 import org.lsst.ccs.localdb.statusdb.model.DataMetaData;
36 import org.lsst.ccs.localdb.statusdb.model.RawData;
37 import org.lsst.ccs.localdb.statusdb.model.StatData;
38 import org.lsst.ccs.localdb.statusdb.model.StatDesc;
39 import org.lsst.ccs.localdb.statusdb.server.ChannelMetaData;
40 import org.lsst.ccs.localdb.statusdb.server.Data;
41 import org.lsst.ccs.localdb.statusdb.server.DataChannel;
42 import org.lsst.ccs.localdb.statusdb.server.TrendingData;
43 import org.lsst.ccs.localdb.statusdb.server.TrendingData.AxisValue;
44 import org.lsst.ccs.localdb.statusdb.server.TrendingData.DataValue;
45 import org.lsst.ccs.localdb.statusdb.utils.StatusdbUtils;
46 import org.lsst.ccs.utilities.logging.Log4JConfiguration;
47 import org.lsst.ccs.utilities.logging.Logger;
48
49
50
51
52
53
54
55 @SuppressWarnings("restriction")
56 @Path("/dataserver")
57 @Singleton
58 public class DataServer {
59
60 private static SessionFactory fac;
61 static {
62
63
64
65 Log4JConfiguration.initialize();
66
67 Logger.configure();
68 }
69 static Logger log = Logger.getLogger("org.lsst.ccs.localdb");
70
71 public DataServer() {
72
73 Properties p = new Properties();
74 String hcdName = "hibernate.connection.datasource";
75
76 try {
77 Context env = new InitialContext();
78 try {
79 if (env.lookup("jdbc/css") != null) {
80 p.setProperty(hcdName, "jdbc/ccs");
81 }
82 }
83 catch (NamingException e) {
84 }
85 try {
86 if (env.lookup("java:comp/env/jdbc/ccs") != null) {
87 p.setProperty(hcdName, "java:comp/env/jdbc/ccs");
88 }
89 }
90 catch (NamingException e) {
91 }
92
93
94
95 }
96 catch (NamingException ne) {
97 }
98
99 fac = StatusdbUtils.getSessionFactory(p);
100
101 log.info("Starting Data Server");
102
103 }
104
105 private List<DataChannel> getListOfChannels() {
106
107 log.info("Loading Channel Information");
108
109 Session sess = fac.openSession();
110 @SuppressWarnings("unchecked")
111 List<DataDesc> l = sess.createQuery("from DataDesc order by srcName").list();
112
113 ArrayList<DataChannel> channels = new ArrayList<>();
114 log.info("Done with the query "+l.size());
115 for (DataDesc d : l) {
116 DataChannel dc = new DataChannel(d);
117 dc.getMetadata().put("subsystem", d.getSrcSubsystem());
118 channels.add(dc);
119 log.debug("retrieving Data Channel path= " + dc.getPathAsString());
120 }
121
122 sess.close();
123 return channels;
124 }
125
126
127
128
129
130
131
132
133 @GET
134 @Path("/data/{id}")
135 public Data getData(@PathParam("id") long id,
136 @QueryParam("t1")
137 @DefaultValue("-1") long t1,
138 @QueryParam("t2")
139 @DefaultValue("-1") long t2,
140 @QueryParam("flavor") String flavor,
141 @QueryParam("n")
142 @DefaultValue("30") int nbins) {
143
144
145
146
147
148
149
150 boolean useStat = false;
151 boolean useRaw = false;
152 if ("stat".equals(flavor)) {
153 useStat = true;
154 } else if ("raw".equals(flavor)) {
155 useRaw = true;
156 } else {
157 flavor = "unspecified";
158 }
159
160
161 log.info("request for data " + id + "[" + t1 + "," + t2 + "] "
162 + flavor + " " + nbins);
163
164 if (t2 < 0) {
165 t2 = System.currentTimeMillis();
166 }
167 if (t1 < 0) {
168 t1 = t2 - 3600000L;
169 }
170
171 log.info("request for data " + id + "[" + t1 + "," + t2 + "] "
172 + flavor + " " + nbins);
173
174
175
176 long rawId = id;
177
178 if (useRaw) {
179 log.debug("sending raw data");
180 return exportRawData(rawId, t1, t2);
181 } else if (useStat) {
182
183 StatDesc statSource = null;
184 long statSourceN = -1;
185 Map<StatDesc, Long> stats = getAvailableStats(rawId, t1, t2);
186
187 log.info("stats :");
188 for (Map.Entry<StatDesc, Long> s : stats.entrySet()) {
189 log.info(s);
190 log.info(" " + s.getKey().getId() + " "
191 + s.getKey().getTimeBinWidth() + " : " + s.getValue());
192 }
193
194
195 for (Map.Entry<StatDesc, Long> s : stats.entrySet()) {
196 long n = s.getValue();
197 if (n > nbins / 2) {
198
199
200
201
202 if (statSource != null) {
203 if (n < statSourceN) {
204 statSource = s.getKey();
205 statSourceN = n;
206 }
207 } else {
208 statSource = s.getKey();
209 statSourceN = n;
210 }
211 }
212 }
213 if (statSource != null) {
214
215 log.debug("sending stat from stat sampling "
216 + statSource.getTimeBinWidth() + " nsamples "
217 + statSourceN);
218 return exportStatDataFromStat(rawId, statSource.getId(), t1, t2, nbins);
219 }
220 }
221
222 log.debug("sending stat from raw");
223 return exportStatDataFromRaw(rawId, t1, t2, nbins);
224 }
225
226 public List<ChannelMetaData> getMetadata(int channelId) {
227 return getMetadata(channelId, -1, -1);
228 }
229
230 public List<ChannelMetaData> getMetadata(int channelId, long t1, long t2) {
231
232 Session sess = fac.openSession();
233 String queryStr = "from DataMetaData where rawDescr_id = :id ";
234 if (t1 > -1) {
235 queryStr += "and (tstopmillis > :t1 or tstopmillis = -1) ";
236 }
237 if (t2 > -1) {
238 queryStr += "and tstartmillis < :t2 ";
239 }
240
241 Query q = sess.createQuery(queryStr);
242 q.setParameter("id", channelId);
243 if (t1 > -1) {
244 q.setParameter("t1", t1);
245 }
246 if (t2 > -1) {
247 q.setParameter("t2", t2);
248 }
249
250 @SuppressWarnings("unchecked")
251 List<DataMetaData> l = q.list();
252
253 sess.close();
254 List<ChannelMetaData> out = new ArrayList<ChannelMetaData>();
255 for (DataMetaData md : l) {
256 out.add(new ChannelMetaData(md));
257 }
258 return out;
259
260
261 }
262
263 @XmlRootElement(name = "channelinfo")
264 public static class ChannelMetadataList {
265
266 @XmlElementWrapper(name = "channelmetadata")
267 @XmlElement(name = "channelmetadatavalue")
268 public List<ChannelMetaData> list;
269
270 public ChannelMetadataList(List<ChannelMetaData> list) {
271 this.list = list;
272 }
273
274 public ChannelMetadataList() {
275 }
276 }
277
278 @GET
279 @Path("/channelinfo/{id}")
280 public DataServer.ChannelMetadataList getMetadataList(@PathParam("id") long channelId) {
281 long rawId = channelId;
282 return new DataServer.ChannelMetadataList(getMetadata((int) rawId));
283 }
284
285
286
287
288
289 @GET
290 @Path("/listchannels")
291 public DataChannel.DataChannelList getChannels() {
292 return new DataChannel.DataChannelList(getListOfChannels());
293 }
294
295
296
297
298
299
300 @GET
301 @Path("/listchannels/{subsystem}")
302 public DataChannel.DataChannelList getChannels(
303 @PathParam("subsystem") String subsystemName) {
304 List<DataChannel> channels = getListOfChannels();
305 ArrayList<DataChannel> subChannels = new ArrayList<DataChannel>();
306 for (DataChannel dc : channels) {
307 if (dc.getPath()[0].equals(subsystemName)) {
308 subChannels.add(dc);
309 }
310 }
311 return new DataChannel.DataChannelList(subChannels);
312
313 }
314
315
316
317
318
319
320
321 public DataChannel[] getChannels(String partialPath, int level) {
322
323 return null;
324 }
325
326
327
328
329
330
331
332 public DataChannel[] getChannelsByKeywork(String keyword) {
333
334 return null;
335
336
337
338 }
339
340 protected Map<StatDesc, Long> getAvailableStats(long rawId, long t1, long t2) {
341 Session sess = fac.openSession();
342 Map<StatDesc, Long> m = new HashMap<StatDesc, Long>();
343 Query q = sess
344 .createQuery("select d, count(x) from StatDesc d, StatData x where d.rawDescr.id = :id "
345 + "and x.descr = d and x.tstampFirst >= :t1 and x.tstampLast <= :t2 order by d.timeBinWidth");
346 q.setParameter("id", rawId);
347 q.setParameter("t1", t1);
348 q.setParameter("t2", t2);
349 @SuppressWarnings("unchecked")
350 List<Object[]> l = q.list();
351 for (Object[] r : l) {
352 if (r[0] == null) {
353 continue;
354 }
355 m.put((StatDesc) r[0], (Long) r[1]);
356 }
357 sess.close();
358 return m;
359 }
360
361 protected long getAvailableRawData(long rawId, long t1, long t2) {
362 Session sess = fac.openSession();
363 Query q = sess
364 .createQuery("select count(r) from RawData r where r.descr.id = :id and r.tstamp between :t1 and :t2 order by r.tstamp");
365 q.setParameter("id", rawId);
366 q.setParameter("t1", t1);
367 q.setParameter("t2", t2);
368 long n = (Long) q.uniqueResult();
369 sess.close();
370 return n;
371 }
372
373 protected List<RawData> getRawData(long id, long t1, long t2) {
374 Session sess = fac.openSession();
375
376 Query q = sess
377 .createQuery("from RawData r where r.descr.id = :id and r.tstamp between :t1 and :t2 order by r.tstamp");
378 q.setParameter("id", id);
379 q.setParameter("t1", t1);
380 q.setParameter("t2", t2);
381 @SuppressWarnings("unchecked")
382 List<RawData> l = q.list();
383 log.debug("retrieved raw data " + id + "[" + t1 + "," + t2 + "] : "
384 + l.size());
385 sess.close();
386
387 return l;
388 }
389
390 protected List<StatData> getStatData(long id, long t1, long t2) {
391 Session sess = fac.openSession();
392
393 Query q = sess
394 .createQuery("from StatData r where r.descr.id = :id and r.tstampFirst >= :t1 and r.tstampLast <= :t2 order by r.tstampFirst");
395 q.setParameter("id", id);
396 q.setParameter("t1", t1);
397 q.setParameter("t2", t2);
398 @SuppressWarnings("unchecked")
399 List<StatData> l = q.list();
400 log.debug("retrieved stat data " + id + "[" + t1 + "," + t2 + "] : "
401 + l.size());
402 sess.close();
403 return l;
404 }
405
406 protected Data exportRawData(long rawId, long t1, long t2) {
407 List<RawData> l = getRawData(rawId, t1, t2);
408
409 Data d = new Data();
410 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
411
412 TrendingData[] data = new TrendingData[l.size()];
413 for (int i = 0; i < l.size(); i++) {
414 RawData r = l.get(i);
415 TrendingData dt = new TrendingData();
416 data[i] = dt;
417 long tStamp = r.getTstamp();
418 AxisValue axisValue = new AxisValue("time", tStamp);
419 dt.setAxisValue(axisValue);
420 DataValue[] dataValue = new DataValue[1];
421 Double dd = r.getDoubleData();
422 dataValue[0] = new DataValue("value", dd == null ? 0 : dd);
423 dt.setDataValue(dataValue);
424 }
425 d.getTrendingResult().setTrendingDataArray(data);
426
427 return d;
428 }
429
430 protected Data exportStatDataFromRaw(long rawId, long t1, long t2, int nsamples) {
431
432 Session sess = fac.openSession();
433
434 SQLQuery q = sess
435 .createSQLQuery("select tlow, thigh, datasum/entries as mean, "
436 + " sqrt((datasumsquared - datasum*datasum/entries)/(entries-1)) as rms "
437 + " from ( SELECT MIN(rd.tstampmills) AS tlow, MAX(rd.tstampmills) AS thigh, "
438 + " SUM(rd.doubleData) AS datasum, SUM(rd.doubleData*rd.doubleData) AS datasumsquared, "
439 + " count(1) AS entries from rawdata rd where descr_id = :id and tstampmills >= :t1 "
440 + " and tstampmills <= :t2 group by floor(rd.tstampmills/:deltat) ) accumulated where entries > 0 ");
441
442 long deltat = (t2 - t1) / nsamples;
443
444 q.setParameter("id", rawId);
445 q.setParameter("t1", t1);
446 q.setParameter("t2", t2);
447 q.setParameter("deltat", deltat);
448 @SuppressWarnings("unchecked")
449 List<Object[]> l = q.list();
450
451 sess.close();
452
453 Data d = new Data();
454 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
455
456 d.getTrendingResult().setTrendingDataArray(new TrendingData[l.size()]);
457 int count = 0;
458 for (Object[] obj : l) {
459
460 long low = ((BigInteger) obj[0]).longValue();
461 long high = ((BigInteger) obj[1]).longValue();
462 TrendingData dt = new TrendingData();
463 d.getTrendingResult().getTrendingDataArray()[count++] = dt;
464 dt.setAxisValue(new AxisValue("time", (low + high) / 2, low, high));
465 DataValue[] dataValue = new DataValue[2];
466
467 double value = ((Double) obj[2]).doubleValue();
468 double rms = 0;
469 if (obj[3] != null) {
470 rms = ((Double) obj[3]).doubleValue();
471 }
472 dataValue[0] = new DataValue("value", value);
473 dataValue[1] = new DataValue("rms", rms);
474 dt.setDataValue(dataValue);
475 }
476
477 return d;
478 }
479
480 protected Data exportStatDataFromStat(long rawId, long id, long t1, long t2,
481 int nsamples) {
482 List<StatData> in = getStatData(id, t1, t2);
483
484 int n = in.size();
485 int rebin = 1;
486 int nout = n;
487 if (n > nsamples * 3) {
488 rebin = n / nsamples;
489 nout = (n + rebin - 1) / rebin;
490 log.debug("will rebin stat by " + rebin + " : " + nout);
491 }
492
493 Data d = new Data();
494 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
495
496 d.getTrendingResult().setTrendingDataArray(new TrendingData[nout]);
497 int iout = 0;
498 int ibin = 0;
499 double sum = 0;
500 double s2 = 0;
501 long nsamp = 0;
502 long low = 0;
503 for (StatData sd : in) {
504 sum += sd.getSum();
505 s2 += sd.getSum2();
506 nsamp += sd.getN();
507 if (ibin == 0) {
508 low = sd.getTstampFirst();
509 }
510 if (ibin == rebin - 1 || (iout * rebin + ibin == n - 1)) {
511 log.debug("storing for " + iout + " from " + (ibin + 1)
512 + " bins");
513 TrendingData dt = new TrendingData();
514 d.getTrendingResult().getTrendingDataArray()[iout] = dt;
515 dt.setAxisValue(new AxisValue("time",
516 (low + sd.getTstampLast()) / 2, low, sd.getTstampLast()));
517 DataValue[] dataValue = new DataValue[2];
518 dataValue[0] = new DataValue("value", sum / n);
519 dataValue[1] = new DataValue("rms", s2 / n - (sum / n)
520 * (sum / n));
521 dt.setDataValue(dataValue);
522 iout++;
523 ibin = 0;
524 } else {
525 ibin++;
526 }
527 }
528
529 return d;
530 }
531
532
533 public static void main(String[] args) throws IOException {
534 class MyApplication extends Application {
535
536 @Override
537 public Set<Class<?>> getClasses() {
538 Set<Class<?>> s = new HashSet<Class<?>>();
539 s.add(DataServer.class);
540 return s;
541 }
542 }
543
544 int socket = 8080;
545 if (args.length > 0) {
546 socket = Integer.decode(args[0]);
547 }
548 MyApplication app = new MyApplication();
549 HttpHandler h = RuntimeDelegate.getInstance().createEndpoint(app, HttpHandler.class);
550 HttpServer s = HttpServer.create(new InetSocketAddress(socket), 5);
551 s.createContext("/rest/data/", h);
552 s.start();
553 }
554
555 }