1 package org.lsst.ccs.localdb.statusdb.trendServer;
2
3 import com.sun.jersey.spi.resource.Singleton;
4 import com.sun.net.httpserver.HttpHandler;
5 import com.sun.net.httpserver.HttpServer;
6 import java.io.IOException;
7 import java.math.BigInteger;
8 import java.net.InetSocketAddress;
9 import java.util.ArrayList;
10 import java.util.HashMap;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.Map;
14 import java.util.Properties;
15 import java.util.Set;
16 import javax.naming.Context;
17 import javax.naming.InitialContext;
18 import javax.naming.NamingException;
19 import javax.ws.rs.DefaultValue;
20 import javax.ws.rs.GET;
21
22 import javax.ws.rs.Path;
23 import javax.ws.rs.PathParam;
24 import javax.ws.rs.QueryParam;
25 import javax.ws.rs.core.Application;
26 import javax.ws.rs.ext.RuntimeDelegate;
27 import javax.xml.bind.annotation.XmlElement;
28 import javax.xml.bind.annotation.XmlElementWrapper;
29 import javax.xml.bind.annotation.XmlRootElement;
30 import org.hibernate.Query;
31 import org.hibernate.SQLQuery;
32 import org.hibernate.Session;
33 import org.hibernate.SessionFactory;
34 import org.lsst.ccs.localdb.statusdb.model.DataDesc;
35 import org.lsst.ccs.localdb.statusdb.model.DataMetaData;
36 import org.lsst.ccs.localdb.statusdb.model.RawData;
37 import org.lsst.ccs.localdb.statusdb.model.StatData;
38 import org.lsst.ccs.localdb.statusdb.model.StatDesc;
39 import org.lsst.ccs.localdb.statusdb.server.ChannelMetaData;
40 import org.lsst.ccs.localdb.statusdb.server.Data;
41 import org.lsst.ccs.localdb.statusdb.server.DataChannel;
42 import org.lsst.ccs.localdb.statusdb.server.TrendingData;
43 import org.lsst.ccs.localdb.statusdb.server.TrendingData.AxisValue;
44 import org.lsst.ccs.localdb.statusdb.server.TrendingData.DataValue;
45 import org.lsst.ccs.localdb.statusdb.utils.StatusdbUtils;
46 import org.lsst.ccs.utilities.logging.Logger;
47
48
49
50
51
52
53
54 @SuppressWarnings("restriction")
55 @Path("/dataserver")
56 @Singleton
57 public class DataServer {
58
59 private static SessionFactory fac;
60 static {
61 Logger.configure();
62 }
63 static Logger log = Logger.getLogger("org.lsst.ccs.localdb");
64
65 public DataServer() {
66
67 Properties p = new Properties();
68 String hcdName = "hibernate.connection.datasource";
69
70 try {
71 Context env = new InitialContext();
72 try {
73 if (env.lookup("jdbc/css") != null) {
74 p.setProperty(hcdName, "jdbc/ccs");
75 }
76 }
77 catch (NamingException e) {
78 }
79 try {
80 if (env.lookup("java:comp/env/jdbc/ccs") != null) {
81 p.setProperty(hcdName, "java:comp/env/jdbc/ccs");
82 }
83 }
84 catch (NamingException e) {
85 }
86
87
88
89 }
90 catch (NamingException ne) {
91 }
92
93 fac = StatusdbUtils.getSessionFactory(p);
94
95 log.info("Starting Data Server");
96
97 }
98
99 private List<DataChannel> getListOfChannels() {
100
101 log.info("Loading Channel Information");
102
103 Session sess = fac.openSession();
104 @SuppressWarnings("unchecked")
105 List<DataDesc> l = sess.createQuery("from DataDesc").list();
106
107 ArrayList<DataChannel> channels = new ArrayList<>();
108 log.info("Done with the query "+l.size());
109 for (DataDesc d : l) {
110 DataChannel dc = new DataChannel(d);
111 dc.getMetadata().put("subsystem", d.getSrcSubsystem());
112 channels.add(dc);
113 log.debug("retrieving Data Channel path= " + dc.getPathAsString());
114 }
115
116 sess.close();
117 return channels;
118 }
119
120
121
122
123
124
125
126
127 @GET
128 @Path("/data/{id}")
129 public Data getData(@PathParam("id") long id,
130 @QueryParam("t1")
131 @DefaultValue("-1") long t1,
132 @QueryParam("t2")
133 @DefaultValue("-1") long t2,
134 @QueryParam("flavor") String flavor,
135 @QueryParam("n")
136 @DefaultValue("30") int nbins) {
137
138
139
140
141
142
143
144 boolean useStat = false;
145 boolean useRaw = false;
146 if ("stat".equals(flavor)) {
147 useStat = true;
148 } else if ("raw".equals(flavor)) {
149 useRaw = true;
150 } else {
151 flavor = "unspecified";
152 }
153
154
155 log.info("request for data " + id + "[" + t1 + "," + t2 + "] "
156 + flavor + " " + nbins);
157
158 if (t2 < 0) {
159 t2 = System.currentTimeMillis();
160 }
161 if (t1 < 0) {
162 t1 = t2 - 3600000L;
163 }
164
165 log.info("request for data " + id + "[" + t1 + "," + t2 + "] "
166 + flavor + " " + nbins);
167
168
169
170 long rawId = id;
171
172 if (useRaw) {
173 log.debug("sending raw data");
174 return exportRawData(rawId, t1, t2);
175 } else if (useStat) {
176
177 StatDesc statSource = null;
178 long statSourceN = -1;
179 Map<StatDesc, Long> stats = getAvailableStats(rawId, t1, t2);
180
181 log.info("stats :");
182 for (Map.Entry<StatDesc, Long> s : stats.entrySet()) {
183 log.info(s);
184 log.info(" " + s.getKey().getId() + " "
185 + s.getKey().getTimeBinWidth() + " : " + s.getValue());
186 }
187
188
189 for (Map.Entry<StatDesc, Long> s : stats.entrySet()) {
190 long n = s.getValue();
191 if (n > nbins / 2) {
192
193
194
195
196 if (statSource != null) {
197 if (n < statSourceN) {
198 statSource = s.getKey();
199 statSourceN = n;
200 }
201 } else {
202 statSource = s.getKey();
203 statSourceN = n;
204 }
205 }
206 }
207 if (statSource != null) {
208
209 log.debug("sending stat from stat sampling "
210 + statSource.getTimeBinWidth() + " nsamples "
211 + statSourceN);
212 return exportStatDataFromStat(rawId, statSource.getId(), t1, t2, nbins);
213 }
214 }
215
216 log.debug("sending stat from raw");
217 return exportStatDataFromRaw(rawId, t1, t2, nbins);
218 }
219
220 public List<ChannelMetaData> getMetadata(int channelId) {
221 return getMetadata(channelId, -1, -1);
222 }
223
224 public List<ChannelMetaData> getMetadata(int channelId, long t1, long t2) {
225
226 Session sess = fac.openSession();
227 String queryStr = "from DataMetaData where rawDescr_id = :id ";
228 if (t1 > -1) {
229 queryStr += "and (tstopmillis > :t1 or tstopmillis = -1) ";
230 }
231 if (t2 > -1) {
232 queryStr += "and tstartmillis < :t2 ";
233 }
234
235 Query q = sess.createQuery(queryStr);
236 q.setParameter("id", channelId);
237 if (t1 > -1) {
238 q.setParameter("t1", t1);
239 }
240 if (t2 > -1) {
241 q.setParameter("t2", t2);
242 }
243
244 @SuppressWarnings("unchecked")
245 List<DataMetaData> l = q.list();
246
247 sess.close();
248 List<ChannelMetaData> out = new ArrayList<ChannelMetaData>();
249 for (DataMetaData md : l) {
250 out.add(new ChannelMetaData(md));
251 }
252 return out;
253
254
255 }
256
257 @XmlRootElement(name = "channelinfo")
258 public static class ChannelMetadataList {
259
260 @XmlElementWrapper(name = "channelmetadata")
261 @XmlElement(name = "channelmetadatavalue")
262 public List<ChannelMetaData> list;
263
264 public ChannelMetadataList(List<ChannelMetaData> list) {
265 this.list = list;
266 }
267
268 public ChannelMetadataList() {
269 }
270 }
271
272 @GET
273 @Path("/channelinfo/{id}")
274 public DataServer.ChannelMetadataList getMetadataList(@PathParam("id") long channelId) {
275 long rawId = channelId;
276 return new DataServer.ChannelMetadataList(getMetadata((int) rawId));
277 }
278
279
280
281
282
283 @GET
284 @Path("/listchannels")
285 public DataChannel.DataChannelList getChannels() {
286 return new DataChannel.DataChannelList(getListOfChannels());
287 }
288
289
290
291
292
293
294 @GET
295 @Path("/listchannels/{subsystem}")
296 public DataChannel.DataChannelList getChannels(
297 @PathParam("subsystem") String subsystemName) {
298 List<DataChannel> channels = getListOfChannels();
299 ArrayList<DataChannel> subChannels = new ArrayList<DataChannel>();
300 for (DataChannel dc : channels) {
301 if (dc.getPath()[0].equals(subsystemName)) {
302 subChannels.add(dc);
303 }
304 }
305 return new DataChannel.DataChannelList(subChannels);
306
307 }
308
309
310
311
312
313
314
315 public DataChannel[] getChannels(String partialPath, int level) {
316
317 return null;
318 }
319
320
321
322
323
324
325
326 public DataChannel[] getChannelsByKeywork(String keyword) {
327
328 return null;
329
330
331
332 }
333
334 protected Map<StatDesc, Long> getAvailableStats(long rawId, long t1, long t2) {
335 Session sess = fac.openSession();
336 Map<StatDesc, Long> m = new HashMap<StatDesc, Long>();
337 Query q = sess
338 .createQuery("select d, count(x) from StatDesc d, StatData x where d.rawDescr.id = :id "
339 + "and x.descr = d and x.tstampFirst >= :t1 and x.tstampLast <= :t2 order by d.timeBinWidth");
340 q.setParameter("id", rawId);
341 q.setParameter("t1", t1);
342 q.setParameter("t2", t2);
343 @SuppressWarnings("unchecked")
344 List<Object[]> l = q.list();
345 for (Object[] r : l) {
346 if (r[0] == null) {
347 continue;
348 }
349 m.put((StatDesc) r[0], (Long) r[1]);
350 }
351 sess.close();
352 return m;
353 }
354
355 protected long getAvailableRawData(long rawId, long t1, long t2) {
356 Session sess = fac.openSession();
357 Query q = sess
358 .createQuery("select count(r) from RawData r where r.descr.id = :id and r.tstamp between :t1 and :t2 order by r.tstamp");
359 q.setParameter("id", rawId);
360 q.setParameter("t1", t1);
361 q.setParameter("t2", t2);
362 long n = (Long) q.uniqueResult();
363 sess.close();
364 return n;
365 }
366
367 protected List<RawData> getRawData(long id, long t1, long t2) {
368 Session sess = fac.openSession();
369
370 Query q = sess
371 .createQuery("from RawData r where r.descr.id = :id and r.tstamp between :t1 and :t2 order by r.tstamp");
372 q.setParameter("id", id);
373 q.setParameter("t1", t1);
374 q.setParameter("t2", t2);
375 @SuppressWarnings("unchecked")
376 List<RawData> l = q.list();
377 log.debug("retrieved raw data " + id + "[" + t1 + "," + t2 + "] : "
378 + l.size());
379 sess.close();
380
381 return l;
382 }
383
384 protected List<StatData> getStatData(long id, long t1, long t2) {
385 Session sess = fac.openSession();
386
387 Query q = sess
388 .createQuery("from StatData r where r.descr.id = :id and r.tstampFirst >= :t1 and r.tstampLast <= :t2 order by r.tstampFirst");
389 q.setParameter("id", id);
390 q.setParameter("t1", t1);
391 q.setParameter("t2", t2);
392 @SuppressWarnings("unchecked")
393 List<StatData> l = q.list();
394 log.debug("retrieved stat data " + id + "[" + t1 + "," + t2 + "] : "
395 + l.size());
396 sess.close();
397 return l;
398 }
399
400 protected Data exportRawData(long rawId, long t1, long t2) {
401 List<RawData> l = getRawData(rawId, t1, t2);
402
403 Data d = new Data();
404 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
405
406 TrendingData[] data = new TrendingData[l.size()];
407 for (int i = 0; i < l.size(); i++) {
408 RawData r = l.get(i);
409 TrendingData dt = new TrendingData();
410 data[i] = dt;
411 long tStamp = r.getTstamp();
412 AxisValue axisValue = new AxisValue("time", tStamp);
413 dt.setAxisValue(axisValue);
414 DataValue[] dataValue = new DataValue[1];
415 Double dd = r.getDoubleData();
416 dataValue[0] = new DataValue("value", dd == null ? 0 : dd);
417 dt.setDataValue(dataValue);
418 }
419 d.getTrendingResult().setTrendingDataArray(data);
420
421 return d;
422 }
423
424 protected Data exportStatDataFromRaw(long rawId, long t1, long t2, int nsamples) {
425
426 Session sess = fac.openSession();
427
428 SQLQuery q = sess
429 .createSQLQuery("select tlow, thigh, datasum/entries as mean, "
430 + " sqrt((datasumsquared - datasum*datasum/entries)/(entries-1)) as rms "
431 + " from ( SELECT MIN(rd.tstampmills) AS tlow, MAX(rd.tstampmills) AS thigh, "
432 + " SUM(rd.doubleData) AS datasum, SUM(rd.doubleData*rd.doubleData) AS datasumsquared, "
433 + " count(1) AS entries from rawdata rd where descr_id = :id and tstampmills >= :t1 "
434 + " and tstampmills <= :t2 group by floor(rd.tstampmills/:deltat) ) accumulated where entries > 0 ");
435
436 long deltat = (t2 - t1) / nsamples;
437
438 q.setParameter("id", rawId);
439 q.setParameter("t1", t1);
440 q.setParameter("t2", t2);
441 q.setParameter("deltat", deltat);
442 @SuppressWarnings("unchecked")
443 List<Object[]> l = q.list();
444
445 sess.close();
446
447 Data d = new Data();
448 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
449
450 d.getTrendingResult().setTrendingDataArray(new TrendingData[l.size()]);
451 int count = 0;
452 for (Object[] obj : l) {
453
454 long low = ((BigInteger) obj[0]).longValue();
455 long high = ((BigInteger) obj[1]).longValue();
456 TrendingData dt = new TrendingData();
457 d.getTrendingResult().getTrendingDataArray()[count++] = dt;
458 dt.setAxisValue(new AxisValue("time", (low + high) / 2, low, high));
459 DataValue[] dataValue = new DataValue[2];
460
461 double value = ((Double) obj[2]).doubleValue();
462 double rms = 0;
463 if (obj[3] != null) {
464 rms = ((Double) obj[3]).doubleValue();
465 }
466 dataValue[0] = new DataValue("value", value);
467 dataValue[1] = new DataValue("rms", rms);
468 dt.setDataValue(dataValue);
469 }
470
471 return d;
472 }
473
474 protected Data exportStatDataFromStat(long rawId, long id, long t1, long t2,
475 int nsamples) {
476 List<StatData> in = getStatData(id, t1, t2);
477
478 int n = in.size();
479 int rebin = 1;
480 int nout = n;
481 if (n > nsamples * 3) {
482 rebin = n / nsamples;
483 nout = (n + rebin - 1) / rebin;
484 log.debug("will rebin stat by " + rebin + " : " + nout);
485 }
486
487 Data d = new Data();
488 d.setDataMetaData(getMetadata((int) rawId, t1, t2));
489
490 d.getTrendingResult().setTrendingDataArray(new TrendingData[nout]);
491 int iout = 0;
492 int ibin = 0;
493 double sum = 0;
494 double s2 = 0;
495 long nsamp = 0;
496 long low = 0;
497 for (StatData sd : in) {
498 sum += sd.getSum();
499 s2 += sd.getSum2();
500 nsamp += sd.getN();
501 if (ibin == 0) {
502 low = sd.getTstampFirst();
503 }
504 if (ibin == rebin - 1 || (iout * rebin + ibin == n - 1)) {
505 log.debug("storing for " + iout + " from " + (ibin + 1)
506 + " bins");
507 TrendingData dt = new TrendingData();
508 d.getTrendingResult().getTrendingDataArray()[iout] = dt;
509 dt.setAxisValue(new AxisValue("time",
510 (low + sd.getTstampLast()) / 2, low, sd.getTstampLast()));
511 DataValue[] dataValue = new DataValue[2];
512 dataValue[0] = new DataValue("value", sum / n);
513 dataValue[1] = new DataValue("rms", s2 / n - (sum / n)
514 * (sum / n));
515 dt.setDataValue(dataValue);
516 iout++;
517 ibin = 0;
518 } else {
519 ibin++;
520 }
521 }
522
523 return d;
524 }
525
526
527 public static void main(String[] args) throws IOException {
528 class MyApplication extends Application {
529
530 @Override
531 public Set<Class<?>> getClasses() {
532 Set<Class<?>> s = new HashSet<Class<?>>();
533 s.add(DataServer.class);
534 return s;
535 }
536 }
537
538 int socket = 8080;
539 if (args.length > 0) {
540 socket = Integer.decode(args[0]);
541 }
542 MyApplication app = new MyApplication();
543 HttpHandler h = RuntimeDelegate.getInstance().createEndpoint(app, HttpHandler.class);
544 HttpServer s = HttpServer.create(new InetSocketAddress(socket), 5);
545 s.createContext("/rest/data/", h);
546 s.start();
547 }
548
549 }