├── LICENSE ├── README.md ├── bin ├── compute-kira-classpath.sh └── compute-kira-jars.sh ├── lib ├── README.md └── jfits-0.94.jar ├── pom.xml ├── resources ├── corrdir │ ├── hdu0_2mass-atlas-990214n-j1100244.fits │ ├── hdu0_2mass-atlas-990214n-j1100256.fits │ ├── hdu0_2mass-atlas-990214n-j1110021.fits │ ├── hdu0_2mass-atlas-990214n-j1110032.fits │ ├── hdu0_2mass-atlas-990214n-j1180244.fits │ ├── hdu0_2mass-atlas-990214n-j1180256.fits │ ├── hdu0_2mass-atlas-990214n-j1190021.fits │ ├── hdu0_2mass-atlas-990214n-j1190032.fits │ ├── hdu0_2mass-atlas-990214n-j1200244.fits │ └── hdu0_2mass-atlas-990214n-j1200256.fits └── template.hdr ├── scratch ├── README.md ├── pyspark │ └── src │ │ └── main │ │ └── python │ │ ├── kira.py │ │ └── repro.py ├── scala │ ├── Background.c │ ├── Background.h │ ├── Background.scala │ ├── Extractor.c │ ├── Extractor.h │ ├── Extractor.scala │ ├── README.md │ ├── Sepbackmap.scala │ ├── Sepobj.scala │ ├── Utils.scala │ ├── libBackgroundImpl.jnilib │ ├── libs │ │ ├── include │ │ │ └── sep.h │ │ ├── jfits-0.94.jar │ │ └── libsep.a │ └── test.c ├── spark-ec2 │ ├── README.md │ ├── bin │ │ ├── compute-kira-classpath.sh │ │ ├── compute-kira-jars.sh │ │ ├── kira-submit │ │ └── kira-submit-local-jar │ ├── data │ │ ├── image1.fits │ │ ├── image2.fits │ │ ├── image3.fits │ │ └── image4.fits │ ├── etc │ │ ├── download-gluster.sh │ │ ├── ec2 │ │ │ ├── run-local.sh │ │ │ └── run-para.sh │ │ ├── edison │ │ │ ├── run-local.sh │ │ │ └── run-para.sh │ │ ├── pom.xml-spark-1.2.2-SNAPSHOT │ │ ├── run.sh │ │ ├── setup-gluster.sh │ │ └── spark-env.sh │ ├── libs │ │ ├── include │ │ │ └── sep.h │ │ ├── jfits-0.94.jar │ │ ├── libBackgroundImpl.so │ │ ├── libsep.so │ │ ├── libsep.so.0 │ │ └── libsep.so.0.0.0 │ ├── pom.xml │ └── src │ │ └── main │ │ ├── C │ │ ├── Background.c │ │ ├── Background.h │ │ ├── Background_Sepbackmap.h │ │ ├── Background_Sepobj.h │ │ ├── Extractor.c │ │ ├── Extractor.h │ │ ├── README.md │ │ └── test.c │ │ └── scala │ │ ├── Background.scala │ │ ├── Extractor.scala │ │ ├── Kira.scala │ │ ├── Sepbackmap.scala │ │ ├── Sepobj.scala │ │ └── Utils.scala ├── spark │ ├── README.md │ ├── bin │ │ ├── compute-kira-classpath.sh │ │ ├── compute-kira-jars.sh │ │ └── kira-submit │ ├── data │ │ ├── image1.fits │ │ ├── image2.fits │ │ ├── image3.fits │ │ └── image4.fits │ ├── libs │ │ ├── include │ │ │ └── sep.h │ │ ├── jfits-0.94.jar │ │ └── libsep.a │ ├── pom.xml │ └── src │ │ └── main │ │ ├── C │ │ ├── Background.c │ │ ├── Background.h │ │ ├── Background_Sepbackmap.h │ │ ├── Background_Sepobj.h │ │ ├── Extractor.c │ │ ├── Extractor.h │ │ ├── README.md │ │ └── test.c │ │ └── scala │ │ ├── Background.scala │ │ ├── Extractor.scala │ │ ├── Kira.scala │ │ ├── Sepbackmap.scala │ │ ├── Sepobj.scala │ │ ├── Utils.scala │ │ └── libBackgroundImpl.jnilib └── test │ ├── Background.c │ ├── Background.h │ ├── Background.java │ ├── Background_Sepbackmap.h │ ├── Background_Sepobj.h │ ├── Fits.java │ ├── libBackgroundImpl.jnilib │ ├── libs │ ├── include │ │ └── sep.h │ ├── jfits-0.94.jar │ └── libsep.a │ └── test.c ├── src └── main │ ├── resources │ └── avro │ │ └── fits.avdl │ └── scala │ └── edu │ └── berkeley │ └── bids │ └── kira │ ├── Fits2Kira.scala │ ├── models │ ├── Coordinate.scala │ ├── Fits.scala │ ├── FitsMetadata.scala │ └── Template.scala │ ├── serialization │ └── MaddKryoRegistrator.scala │ └── util │ └── FitsUtils.scala └── test └── scala └── README.md /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 ZHAO ZHANG 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kira 2 | ============ 3 | 4 | # Building 5 | 6 | We use maven to build. You can build with: 7 | 8 | ``` 9 | mvn clean package 10 | ``` 11 | 12 | We require a local jar to be installed the first time you build. We provide this jar in the repository. Install this jar with: 13 | 14 | ``` 15 | mvn install:install-file -Dfile=lib/jfits-0.94.jar -DgroupId="org.esa" -DartifactId="fits" -Dpackaging="jar" -Dversion="0.94" 16 | ``` 17 | 18 | # Spark Fits2Kira 19 | 20 | The spark version has a provided submit script. To run, do: 21 | 22 | ``` 23 | bin/kira-submit 24 | ``` 25 | 26 | You will need to have a copy of the Spark binaries. Set `$SPARK_HOME` to the path to the binaries. -------------------------------------------------------------------------------- /bin/compute-kira-classpath.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Next three commands set CLASSPATH like appassembler 24 | BASEDIR="$KIRA_REPO"/target/appassembler 25 | REPO="$BASEDIR"/repo 26 | if [ ! -f "$BASEDIR"/bin/Fits2Kira ]; then 27 | echo "Failed to find appassembler scripts in $BASEDIR/bin" 28 | echo "You need to build Madd before running this program" 29 | exit 1 30 | fi 31 | eval $(cat "$BASEDIR"/bin/Fits2Kira | grep "^CLASSPATH") 32 | 33 | echo "$CLASSPATH" 34 | -------------------------------------------------------------------------------- /bin/compute-kira-jars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | CLASSPATH=$("$KIRA_REPO"/bin/compute-kira-classpath.sh) 24 | 25 | # list of jars to ship with spark; trim off the first and last from the CLASSPATH 26 | # TODO: brittle? assumes appassembler always puts the $BASE/etc first and the CLI jar last 27 | KIRA_JARS=$(echo "$CLASSPATH" | tr ":" "," | cut -d "," -f 2- | rev | cut -d "," -f 2- | rev) 28 | 29 | echo "$KIRA_JARS" 30 | -------------------------------------------------------------------------------- /lib/README.md: -------------------------------------------------------------------------------- 1 | SparkMontage 2 | ============ 3 | -------------------------------------------------------------------------------- /lib/jfits-0.94.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/lib/jfits-0.94.jar -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1100244.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1100244.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1100256.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1100256.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1110021.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1110021.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1110032.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1110032.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1180244.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1180244.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1180256.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1180256.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1190021.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1190021.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1190032.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1190032.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1200244.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1200244.fits -------------------------------------------------------------------------------- /resources/corrdir/hdu0_2mass-atlas-990214n-j1200256.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/resources/corrdir/hdu0_2mass-atlas-990214n-j1200256.fits -------------------------------------------------------------------------------- /resources/template.hdr: -------------------------------------------------------------------------------- 1 | SIMPLE = T 2 | BITPIX = -64 3 | NAXIS = 2 4 | NAXIS1 = 2259 5 | NAXIS2 = 2199 6 | CTYPE1 = 'RA---TAN' 7 | CTYPE2 = 'DEC--TAN' 8 | CRVAL1 = 210.835222357 9 | CRVAL2 = 54.367562188 10 | CDELT1 = -0.000277780 11 | CDELT2 = 0.000277780 12 | CRPIX1 = 1130.0000 13 | CRPIX2 = 1100.0000 14 | CROTA2 = -0.052834592 15 | HISTORY = MONTAGE TUTORIAL 16 | END 17 | -------------------------------------------------------------------------------- /scratch/README.md: -------------------------------------------------------------------------------- 1 | This directory is for experimental code for JNI. 2 | 3 | 1. Compile Background.java 4 | javac -cp libs/jfits-0.94.jar Fits.java Background.java 5 | 6 | 2. Compile JNI interface 7 | gcc Background.c -o libBackgroundImpl.jnilib -lc -shared -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include" -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include/darwin" -L"/Users/zhaozhang/projects/scratch/java/test/libs" -lsep -I"/Users/zhaozhang/projects/scratch/java/test/libs/include" 8 | 9 | 3. Run the test 10 | java -cp .:libs/jfits-0.94.jar Background 11 | 12 | -------------------------------------------------------------------------------- /scratch/pyspark/src/main/python/kira.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | import numpy as np 19 | import sep 20 | import sys 21 | from operator import add 22 | 23 | from pyspark import SparkContext 24 | 25 | def extract(data): 26 | bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3) 27 | bkg.subfrom(data) 28 | objs = sep.extract(data, 1.5*bkg.globalrms) 29 | flux, fluxerr, flag = sep.sum_circle(data, objs['x'], objs['y'], 5., 30 | err=bkg.globalrms) 31 | kr, flag = sep.kron_radius(data, objs['x'], objs['y'], objs['a'], 32 | objs['b'], objs['theta'], 6.0) 33 | eflux, efluxerr, eflag = sep.sum_ellipse(data, objs['x'], objs['y'], 34 | objs['a'], objs['b'], 35 | objs['theta'], r=2.5 * kr, 36 | err=bkg.globalrms, subpix=1) 37 | retstr = "" 38 | for i in range(len(objs['x'])): 39 | retstr = retstr+(str(objs['x'][i])+"\t"+str(objs['y'][i])+"\t"+str(flux[i])+"\t"+str(fluxerr[i])+"\t"+str(kr[i])+"\t"+str(eflux[i])+"\t"+str(efluxerr[i])+"\t"+str(flag[i])+"\n") 40 | return retstr 41 | 42 | if __name__ == "__main__": 43 | sc = SparkContext(appName="SourceExtractor") 44 | rdd = sc.fitsData("/Users/zhaozhang/projects/SDSS/data") 45 | #rdd = sc.fitsData("/Users/zhaozhang/projects/Kira/scratch/spark-ec2/data/") 46 | catalog = rdd.map(lambda (key, fits): (key, extract(np.copy(fits)))) 47 | catalog.saveAsTextFile("temp-output") 48 | 49 | sc.stop() 50 | -------------------------------------------------------------------------------- /scratch/pyspark/src/main/python/repro.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | import numpy as np 19 | import sep 20 | import sys 21 | from astropy.io import fits 22 | from astropy.wcs import WCS 23 | import reproject 24 | from reproject import reproject_interp 25 | 26 | from pyspark import SparkContext 27 | 28 | if __name__ == "__main__": 29 | sc = SparkContext(appName="Reproject") 30 | rdd = sc.fitsFiles("/Users/zhaozhang/projects/SDSS/data") 31 | header = rdd.take(1).pop().pop().header 32 | rprordd = rdd.map(lambda x: reproject_interp(x.pop(), header)) 33 | nrdd = rprordd.zipWithIndex() 34 | nrdd.foreach(lambda ((array, footprint), index): fits.writeto(str(index)+".fits", array, header)) 35 | 36 | sc.stop() 37 | -------------------------------------------------------------------------------- /scratch/scala/Background.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "Background.h" 6 | #include "sep.h" 7 | 8 | void sep_back_c_java(JNIEnv *env, jclass cls, jobject obj, jclass bkmapcls, jobject bkmap, sepbackmap *p){ 9 | /*set the globalback and globalrms field as return value*/ 10 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 11 | assert(id_gback != NULL); 12 | (*env)->SetFloatField(env, bkmap, id_gback, p->globalback); 13 | 14 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 15 | assert(id_rms != NULL); 16 | (*env)->SetFloatField(env, bkmap, id_rms, p->globalrms); 17 | 18 | /*set the w and h field as return value*/ 19 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 20 | (*env)->SetIntField(env, bkmap, id_w, p->w); 21 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 22 | (*env)->SetIntField(env, bkmap, id_h, p->h); 23 | 24 | /*set the bw and bh field as return value*/ 25 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 26 | (*env)->SetIntField(env, bkmap, id_bw, p->bw); 27 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 28 | (*env)->SetIntField(env, bkmap, id_bh, p->bh); 29 | 30 | /*set the nx, ny, and n field as return value*/ 31 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 32 | (*env)->SetIntField(env, bkmap, id_nx, p->nx); 33 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 34 | (*env)->SetIntField(env, bkmap, id_ny, p->ny); 35 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 36 | (*env)->SetIntField(env, bkmap, id_n, p->n); 37 | 38 | jmethodID setBackID = (*env)->GetMethodID(env, bkmapcls, "setBack", "([F)V"); 39 | jfloatArray backArray = (*env)->NewFloatArray(env, p->n); 40 | (*env)->SetFloatArrayRegion(env, backArray, 0, p->n, p->back); 41 | (*env)->CallVoidMethod(env, bkmap, setBackID, backArray); 42 | 43 | jmethodID setDbackID = (*env)->GetMethodID(env, bkmapcls, "setDback", "([F)V"); 44 | jfloatArray dbackArray = (*env)->NewFloatArray(env, p->n); 45 | (*env)->SetFloatArrayRegion(env, dbackArray, 0, p->n, p->dback); 46 | (*env)->CallVoidMethod(env, bkmap, setDbackID, dbackArray); 47 | 48 | jmethodID setSigmaID = (*env)->GetMethodID(env, bkmapcls, "setSigma", "([F)V"); 49 | jfloatArray sigmaArray = (*env)->NewFloatArray(env, p->n); 50 | (*env)->SetFloatArrayRegion(env, sigmaArray, 0, p->n, p->sigma); 51 | (*env)->CallVoidMethod(env, bkmap, setSigmaID, sigmaArray); 52 | 53 | jmethodID setDsigmaID = (*env)->GetMethodID(env, bkmapcls, "setDsigma", "([F)V"); 54 | jfloatArray dsigmaArray = (*env)->NewFloatArray(env, p->n); 55 | (*env)->SetFloatArrayRegion(env, dsigmaArray, 0, p->n, p->dsigma); 56 | (*env)->CallVoidMethod(env, bkmap, setDsigmaID, dsigmaArray); 57 | } 58 | 59 | void sep_back_java_c(JNIEnv *env, jclass bkmapcls, jobject bkmap, sepbackmap *p, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 60 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 61 | jint h = (*env)->GetIntField(env, bkmap, id_h); 62 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 63 | jint w = (*env)->GetIntField(env, bkmap, id_w); 64 | 65 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 66 | jint bh = (*env)->GetIntField(env, bkmap, id_bh); 67 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 68 | jint bw = (*env)->GetIntField(env, bkmap, id_bw); 69 | 70 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 71 | jint nx = (*env)->GetIntField(env, bkmap, id_nx); 72 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 73 | jint ny = (*env)->GetIntField(env, bkmap, id_ny); 74 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 75 | jint n = (*env)->GetIntField(env, bkmap, id_n); 76 | 77 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 78 | jfloat gback = (*env)->GetFloatField(env, bkmap, id_gback); 79 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 80 | jfloat rms = (*env)->GetFloatField(env, bkmap, id_rms); 81 | 82 | /*need to implement new array access functions here*/ 83 | jfloat *backarray = (jfloat *)(*env)->GetFloatArrayElements(env, back, NULL); 84 | jfloat *dbackarray = (jfloat *)(*env)->GetFloatArrayElements(env, dback, NULL); 85 | jfloat *sigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, sigma, NULL); 86 | jfloat *dsigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, dsigma, NULL); 87 | 88 | p->h = h; 89 | p->w = w; 90 | p->bh = bh; 91 | p->bw = bw; 92 | p->nx = nx; 93 | p->ny = ny; 94 | p->n = n; 95 | p->globalback = gback; 96 | p->globalrms = rms; 97 | 98 | jfloat *bp = p->back; 99 | jfloat *dbp = p->dback; 100 | jfloat *sp = p->sigma; 101 | jfloat *dsp = p->dsigma; 102 | 103 | memcpy(bp, backarray, sizeof(float)*(p->n)); 104 | memcpy(dbp, dbackarray, sizeof(float)*(p->n)); 105 | memcpy(sp, sigmaarray, sizeof(float)*(p->n)); 106 | memcpy(dsp, dsigmaarray, sizeof(float)*(p->n)); 107 | } 108 | 109 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 110 | (JNIEnv *env, jobject obj, jbyteArray data, jobjectArray maskdata, jint dtype, jint mdtype, jint w, jint h, jint 111 | bw, jint bh, jdouble mthresh, jint fw, jint fh, jdouble fthresh, jobject backmap){ 112 | 113 | int len = (*env)->GetArrayLength(env, data); 114 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 115 | 116 | jboolean *mask; 117 | jbooleanArray dim = NULL; 118 | jbooleanArray oneDim = NULL; 119 | if(maskdata == NULL){ 120 | mask = (void *)NULL; 121 | } 122 | else{ 123 | /*parse the mask matrix*/ 124 | int mlen1 = (*env)->GetArrayLength(env, maskdata); 125 | dim = (jbooleanArray)(*env)->GetObjectArrayElement(env, maskdata, 0); 126 | int mlen2 = (*env)->GetArrayLength(env, dim); 127 | printf("mlen2: %d\n", mlen2); 128 | mask = malloc(sizeof(jboolean)* mlen1 * mlen2); 129 | for(int i=0; iGetObjectArrayElement(env, maskdata, i); 131 | jboolean *element=(*env)->GetBooleanArrayElements(env, oneDim, 0); 132 | memcpy(mask+i*mlen2, element, sizeof(jboolean)*mlen2); 133 | } 134 | } 135 | 136 | 137 | /*printf("%d\n", len); 138 | for(int i=0; iw = w; 144 | p->h = h; 145 | p->bw = bw; 146 | p->bh = bh; 147 | 148 | int status = sep_makeback(array, mask, dtype, mdtype, w, h, bw, bh, mthresh, fw, fh, fthresh, &p); 149 | 150 | //printf("%d\n", status); 151 | printf("C sep_makeback: %d\t%d\t%f\t%f\t%d\t%d\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh, p->nx, p->ny); 152 | printf("C sep_makeback back: %f\t dback: %f\n", *(p->back), *(p->dback)); 153 | 154 | /*Finding the sepbackmap field in the Background class*/ 155 | jclass cls = (*env)->GetObjectClass(env, obj); 156 | assert(cls != NULL); 157 | jfieldID id_bkmap = (*env)->GetFieldID(env, cls, "bkgmap", "LSepbackmap;"); 158 | assert(id_bkmap != NULL); 159 | jobject bkmap = (*env)->GetObjectField(env, obj, id_bkmap); 160 | assert(bkmap != NULL); 161 | jclass bkmapcls = (*env)->GetObjectClass(env, bkmap); 162 | assert(bkmapcls != NULL); 163 | 164 | sep_back_c_java(env, cls, obj, bkmapcls, bkmap, p); 165 | 166 | float *fp = p->back; 167 | for(int i=0; in; i++){ 168 | printf("back: %f\t", *(fp++)); 169 | } 170 | printf("\n"); 171 | 172 | fp = p->dback; 173 | for(int i=0; in; i++){ 174 | printf("dback: %f\t", *(fp++)); 175 | } 176 | printf("\n"); 177 | 178 | fp = p->sigma; 179 | for(int i=0; in; i++){ 180 | printf("sigma: %f\t", *(fp++)); 181 | } 182 | printf("\n"); 183 | 184 | fp = p->dsigma; 185 | for(int i=0; in; i++){ 186 | printf("dsigma: %f\t", *(fp++)); 187 | } 188 | printf("\n"); 189 | free(array); 190 | free(p); 191 | 192 | if(mask != NULL){ 193 | printf("C makeback: free mask\n"); 194 | free(mask); 195 | } 196 | 197 | return status; 198 | } 199 | 200 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 201 | (JNIEnv *env, jobject obj, jobject bkmap, jbyteArray data, jint dtype, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 202 | 203 | int len = (*env)->GetArrayLength(env, data); 204 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 205 | 206 | jclass cls = (*env)->GetObjectClass(env, bkmap); 207 | assert(cls != NULL); 208 | 209 | int backLen = (*env)->GetArrayLength(env, back); 210 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 211 | p->back = malloc(sizeof(float)*backLen); 212 | p->dback = malloc(sizeof(float)*backLen); 213 | p->sigma = malloc(sizeof(float)*backLen); 214 | p->dsigma = malloc(sizeof(float)*backLen); 215 | 216 | sep_back_java_c(env, cls, bkmap, p, back, dback, sigma, dsigma); 217 | 218 | printf("C sep_backarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 219 | printf("C sep_backarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 220 | printf("C sep_backarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 221 | printf("C sep_backarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma)); 222 | 223 | 224 | int status = sep_backarray(p, array, dtype); 225 | printf("status: %d\n", status); 226 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 227 | 228 | printf("%d, \n", array[0]); 229 | 230 | /*for(int i=0; iGetArrayLength(env, data); 246 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 247 | 248 | jclass cls = (*env)->GetObjectClass(env, bkgmap); 249 | assert(cls != NULL); 250 | 251 | int backLen = (*env)->GetArrayLength(env, back); 252 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 253 | p->back = malloc(sizeof(float)*backLen); 254 | p->dback = malloc(sizeof(float)*backLen); 255 | p->sigma = malloc(sizeof(float)*backLen); 256 | p->dsigma = malloc(sizeof(float)*backLen); 257 | 258 | sep_back_java_c(env, cls, bkgmap, p, back, dback, sigma, dsigma); 259 | 260 | printf("C sep_subbackarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 261 | printf("C sep_subbackarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 262 | printf("C sep_subbackarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 263 | printf("C sep_subbackarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma)); 264 | 265 | int status = sep_subbackarray(p, array, dtype); 266 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 267 | 268 | /*printf("C sep_subbackarray: result\n"); 269 | double *dp = (double *)array; 270 | for(int i=0; i 3 | /* Header for class Background */ 4 | 5 | #ifndef _Included_Background 6 | #define _Included_Background 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Background 12 | * Method: sep_makeback 13 | * Signature: ([B[[ZIIIIIIDIIDLBackground/Sepbackmap;)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 16 | (JNIEnv *, jobject, jbyteArray, jobjectArray, jint, jint, jint, jint, jint, jint, jdouble, jint, jint, jdouble, jobject); 17 | 18 | /* 19 | * Class: Background 20 | * Method: sep_backarray 21 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 24 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 25 | 26 | /* 27 | * Class: Background 28 | * Method: sep_backrmsarray 29 | * Signature: (LBackground/Sepbackmap;[Ljava/lang/Object;I)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Background_sep_1backrmsarray 32 | (JNIEnv *, jobject, jobject, jobjectArray, jint); 33 | 34 | /* 35 | * Class: Background 36 | * Method: sep_subbackarray 37 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Background_sep_1subbackarray 40 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 41 | 42 | /* 43 | * Class: Background 44 | * Method: sep_freeback 45 | * Signature: (LBackground/Sepbackmap;)V 46 | */ 47 | JNIEXPORT void JNICALL Java_Background_sep_1freeback 48 | (JNIEnv *, jobject, jobject); 49 | 50 | /* 51 | * Class: Background 52 | * Method: sep_extract 53 | * Signature: ([B[BIISIIFI[BIIIDZD[LBackground/Sepobj;I)I 54 | */ 55 | JNIEXPORT jint JNICALL Java_Background_sep_1extract 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 57 | 58 | /* 59 | * Class: Background 60 | * Method: sep_freeobjarray 61 | * Signature: ([LBackground/Sepobj;I)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Background_sep_1freeobjarray 64 | (JNIEnv *, jobject, jobjectArray, jint); 65 | 66 | /* 67 | * Class: Background 68 | * Method: sep_sum_circle 69 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 70 | */ 71 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circle 72 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 73 | 74 | /* 75 | * Class: Background 76 | * Method: sep_sum_circann 77 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 78 | */ 79 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circann 80 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 81 | 82 | /* 83 | * Class: Background 84 | * Method: sep_sum_ellipse 85 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 86 | */ 87 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipse 88 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 89 | 90 | /* 91 | * Class: Background 92 | * Method: sep_sum_ellipann 93 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 94 | */ 95 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipann 96 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 97 | 98 | /* 99 | * Class: Background 100 | * Method: sep_kron_radius 101 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 102 | */ 103 | JNIEXPORT void JNICALL Java_Background_sep_1kron_1radius 104 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 105 | 106 | /* 107 | * Class: Background 108 | * Method: sep_ellipse_axes 109 | * Signature: ([D[D[D[D[D[D)V 110 | */ 111 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1axes 112 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 113 | 114 | /* 115 | * Class: Background 116 | * Method: sep_ellipse_coeffs 117 | * Signature: ([D[D[D[D[D[D)V 118 | */ 119 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1coeffs 120 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 121 | 122 | /* 123 | * Class: Background 124 | * Method: sep_set_ellipse 125 | * Signature: (Ljava/lang/String;IIDDDDDDS)V 126 | */ 127 | JNIEXPORT void JNICALL Java_Background_sep_1set_1ellipse 128 | (JNIEnv *, jobject, jstring, jint, jint, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jshort); 129 | 130 | /* 131 | * Class: Background 132 | * Method: sep_set_extract_pixstack 133 | * Signature: (I)V 134 | */ 135 | JNIEXPORT void JNICALL Java_Background_sep_1set_1extract_1pixstack 136 | (JNIEnv *, jobject, jint); 137 | 138 | /* 139 | * Class: Background 140 | * Method: sep_get_extract_pixstack 141 | * Signature: ()I 142 | */ 143 | JNIEXPORT jint JNICALL Java_Background_sep_1get_1extract_1pixstack 144 | (JNIEnv *, jobject); 145 | 146 | /* 147 | * Class: Background 148 | * Method: sep_get_errmsg 149 | * Signature: (ILjava/lang/String;)V 150 | */ 151 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errmsg 152 | (JNIEnv *, jobject, jint, jstring); 153 | 154 | /* 155 | * Class: Background 156 | * Method: sep_get_errdetail 157 | * Signature: (Ljava/lang/String;)V 158 | */ 159 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errdetail 160 | (JNIEnv *, jobject, jstring); 161 | 162 | #ifdef __cplusplus 163 | } 164 | #endif 165 | #endif 166 | -------------------------------------------------------------------------------- /scratch/scala/Extractor.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Extractor */ 4 | 5 | #ifndef _Included_Extractor 6 | #define _Included_Extractor 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Extractor 12 | * Method: sep_sum_circle 13 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circle 16 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 17 | 18 | /* 19 | * Class: Extractor 20 | * Method: sep_sum_circann 21 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circann 24 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 25 | 26 | /* 27 | * Class: Extractor 28 | * Method: sep_sum_ellipse 29 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipse 32 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 33 | 34 | /* 35 | * Class: Extractor 36 | * Method: sep_sum_ellipann 37 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipann 40 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 41 | 42 | /* 43 | * Class: Extractor 44 | * Method: sep_extract 45 | * Signature: ([B[BIISIIFI[BIIIDZD[LSepobj;I)I 46 | */ 47 | JNIEXPORT jint JNICALL Java_Extractor_sep_1extract 48 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 49 | 50 | /* 51 | * Class: Extractor 52 | * Method: sep_kron_radius 53 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 54 | */ 55 | JNIEXPORT void JNICALL Java_Extractor_sep_1kron_1radius 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 57 | 58 | /* 59 | * Class: Extractor 60 | * Method: sep_ellipse_coeffs 61 | * Signature: ([D[D[D[D[D[D)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1coeffs 64 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 65 | 66 | /* 67 | * Class: Extractor 68 | * Method: sep_ellipse_axes 69 | * Signature: ([D[D[D[D[D[D)V 70 | */ 71 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1axes 72 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 73 | 74 | /* 75 | * Class: Extractor 76 | * Method: sep_set_ellipse 77 | * Signature: ([BII[D[D[D[D[D[DB)V 78 | */ 79 | JNIEXPORT void JNICALL Java_Extractor_sep_1set_1ellipse 80 | (JNIEnv *, jobject, jbyteArray, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jbyte); 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | #endif 86 | -------------------------------------------------------------------------------- /scratch/scala/README.md: -------------------------------------------------------------------------------- 1 | Compile instructions: 2 | 1. Compile the library: 3 | gcc Background.c Extractor.c -o libBackgroundImpl.jnilib -lc -shared -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include" -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include/darwin" -L"/Users/zhaozhang/projects/scratch/java/test/libs" -lsep -I"/Users/zhaozhang/projects/scratch/java/test/libs/include" 4 | 5 | 2. Compile the Source Extractor 6 | scalac -cp libs/jfits-0.94.jar *.scala 7 | 8 | 3. To execute 9 | scala -cp .:libs/jfits-0.94.jar Test 10 | -------------------------------------------------------------------------------- /scratch/scala/Sepbackmap.scala: -------------------------------------------------------------------------------- 1 | class Sepbackmap(var w:Int, var h:Int, var globalback:Float, var globalrms:Float, var bw:Int, 2 | var bh:Int, var nx:Int, var ny:Int, var n:Int, var back:Array[Float], var dback:Array[Float], 3 | var sigma:Array[Float], var dsigma:Array[Float]){ 4 | back = new Array[Float](n) 5 | dback = new Array[Float](n) 6 | sigma = new Array[Float](n) 7 | dsigma = new Array[Float](n) 8 | 9 | def this(){ 10 | this(0, 0, 0.0F, 0.0F, 0, 0, 0, 0, 0, null, null, null, null) 11 | } 12 | 13 | def setBack(vback: Array[Float]){ 14 | back = new Array[Float](vback.length) 15 | for(i <- (0 until back.length)){ 16 | back(i) = vback(i) 17 | } 18 | } 19 | 20 | def setDback(vdback: Array[Float]){ 21 | dback = new Array[Float](vdback.length) 22 | for(i <- (0 until dback.length)){ 23 | dback(i) = vdback(i) 24 | } 25 | } 26 | 27 | def setSigma(vsigma: Array[Float]){ 28 | sigma = new Array[Float](vsigma.length) 29 | for(i <- (0 until sigma.length)){ 30 | sigma(i) = vsigma(i) 31 | } 32 | } 33 | 34 | def setDsigma(vdsigma: Array[Float]){ 35 | dsigma = new Array[Float](vdsigma.length) 36 | for(i <- (0 until dsigma.length)){ 37 | dsigma(i) = vdsigma(i) 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /scratch/scala/Sepobj.scala: -------------------------------------------------------------------------------- 1 | 2 | class Sepobj(var thresh:Double, var npix:Int, var tnpix:Int, var xmin:Int, var xmax:Int, 3 | var ymin:Int, var ymax:Int, var x:Double, var y:Double, var x2:Double, var y2:Double, 4 | var xy:Double, var a:Float, var b:Float, var theta:Float, var cxx:Float, var cyy:Float, 5 | var cxy:Float, var cflux:Float, var flux:Float, var cpeak:Float, var peak:Float, 6 | var xpeak:Float, var ypeak:Float, var xcpeak:Float, var ycpeak:Float, var flag:Short, var pix:Int) -------------------------------------------------------------------------------- /scratch/scala/Utils.scala: -------------------------------------------------------------------------------- 1 | import java.lang.{Double =>jDouble} 2 | import java.nio.ByteBuffer 3 | import org.eso.fits._ 4 | 5 | object Utils{ 6 | /**Helper function that converts 2d double matrix to 1d byte stream*/ 7 | def flatten(matrix:Array[Array[Double]]):Array[Byte] = { 8 | val h = matrix.length 9 | val w = matrix(0).length 10 | var stream = new Array[Byte](w*h*8) 11 | 12 | for(i <- (0 until h)){ 13 | for(j <- (0 until w)){ 14 | var l: Long = jDouble.doubleToLongBits(matrix(i)(j)) 15 | for(k <- (0 until 8)){ 16 | stream((i*w+j)*8+k) = ((l >>> (k * 8)) & 0xff).toByte 17 | } 18 | } 19 | } 20 | stream 21 | } 22 | 23 | /**Helper function that converts 1d byte stream to 2d double matrix*/ 24 | def deflatten(data:Array[Byte], h:Int, w:Int):Array[Array[Double]] = { 25 | var matrix = Array.ofDim[Double](h, w) 26 | for(i <- (0 until h)){ 27 | for(j <- (0 until w)){ 28 | var bytes = new Array[Byte](8) 29 | for(k <- (0 until 8)){ 30 | bytes(7-k) = (data((i*w+j)*8+k) & 0xff).toByte 31 | } 32 | matrix(i)(j) = ByteBuffer.wrap(bytes).getDouble() 33 | } 34 | } 35 | return matrix 36 | } 37 | 38 | def load(path:String):Array[Array[Double]] = { 39 | val file:FitsFile = new FitsFile(path) 40 | val hdu:FitsHDUnit = file.getHDUnit(0) 41 | val dm:FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 42 | val naxis:Array[Int] = dm.getNaxis() 43 | val ncol = naxis(0) 44 | val nval = dm.getNoValues() 45 | val nrow = nval/ncol 46 | 47 | // build and populate an array 48 | var matrix = Array.ofDim[Float](nrow, ncol) 49 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 50 | 51 | var rmatrix = Array.ofDim[Double](nrow, ncol) 52 | for(i <- (0 until nrow)){ 53 | for(j <- (0 until ncol)){ 54 | rmatrix(i)(j) = matrix(i)(j).toDouble 55 | } 56 | } 57 | 58 | return rmatrix 59 | } 60 | } -------------------------------------------------------------------------------- /scratch/scala/libBackgroundImpl.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/scala/libBackgroundImpl.jnilib -------------------------------------------------------------------------------- /scratch/scala/libs/include/sep.h: -------------------------------------------------------------------------------- 1 | /*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 2 | * 3 | * This file is part of SEP 4 | * 5 | * Copyright 1993-2011 Emmanuel Bertin -- IAP/CNRS/UPMC 6 | * Copyright 2014 SEP developers 7 | * 8 | * SEP is free software: you can redistribute it and/or modify 9 | * it under the terms of the GNU Lesser General Public License as published by 10 | * the Free Software Foundation, either version 3 of the License, or 11 | * (at your option) any later version. 12 | * 13 | * SEP is distributed in the hope that it will be useful, 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | * GNU Lesser General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU Lesser General Public License 19 | * along with SEP. If not, see . 20 | * 21 | *%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/ 22 | 23 | /* datatype codes */ 24 | #define SEP_TBYTE 11 /* 8-bit unsigned byte */ 25 | #define SEP_TINT 31 /* native int type */ 26 | #define SEP_TFLOAT 42 27 | #define SEP_TDOUBLE 82 28 | 29 | /* object & aperture flags */ 30 | #define SEP_OBJ_MERGED 0x0001 /* object is result of deblending */ 31 | #define SEP_OBJ_TRUNC 0x0002 /* object truncated at image boundary */ 32 | #define SEP_OBJ_DOVERFLOW 0x0004 /* not currently used, but could be */ 33 | #define SEP_OBJ_SINGU 0x0008 /* x,y fully correlated */ 34 | #define SEP_APER_TRUNC 0x0010 35 | #define SEP_APER_HASMASKED 0x0020 36 | #define SEP_APER_ALLMASKED 0x0040 37 | #define SEP_APER_NONPOSITIVE 0x0080 38 | 39 | /* input flags for aperture photometry */ 40 | #define SEP_ERROR_IS_VAR 0x0001 41 | #define SEP_ERROR_IS_ARRAY 0x0002 42 | #define SEP_MASK_IGNORE 0x0004 43 | 44 | /*--------------------- global background estimation ------------------------*/ 45 | 46 | typedef struct 47 | { 48 | int w, h; /* original image width, height */ 49 | int bw, bh; /* single tile width, height */ 50 | int nx, ny; /* number of tiles in x, y */ 51 | int n; /* nx*ny */ 52 | float globalback; /* global mean */ 53 | float globalrms; /* global sigma */ 54 | float *back; /* node data for interpolation */ 55 | float *dback; 56 | float *sigma; 57 | float *dsigma; 58 | } sepbackmap; 59 | 60 | int sep_makeback(void *im, /* image data */ 61 | void *mask, /* mask (can be NULL) */ 62 | int dtype, /* image datatype code */ 63 | int mdtype, /* mask datatype code */ 64 | int w, int h, /* image and mask size (width, height) */ 65 | int bw, int bh, /* size of a single background tile */ 66 | float mthresh, /* only (mask<=maskthresh) counted */ 67 | int fw, int fh, /* filter size in tiles */ 68 | float fthresh, /* filter threshold */ 69 | sepbackmap **bkmap); /* OUTPUT */ 70 | /* Create representation of spatially varying image background and variance. 71 | * 72 | * Note that the returned pointer must eventually be freed by calling 73 | * `sep_freeback()`. 74 | * 75 | * If a mask is supplied, only pixels with mask value <= mthresh are counted. 76 | * In addition to the mask, pixels <= -1e30 and NaN are ignored. 77 | * 78 | * Source Extractor defaults: 79 | * 80 | * - bw, bh = (64, 64) 81 | * - fw, fh = (3, 3) 82 | * - fthresh = 0.0 83 | */ 84 | 85 | float sep_globalback(sepbackmap *bkmap); 86 | float sep_globalrms(sepbackmap *bkmap); 87 | /* Get the estimate of the global background "mean" or standard deviation */ 88 | 89 | float sep_backpix_linear(sepbackmap *bkmap, int x, int y); 90 | /* Return background at (x, y). 91 | * Unlike other routines, this uses simple linear interpolation. */ 92 | 93 | int sep_backline(sepbackmap *bkmap, int y, void *line, int dtype); 94 | int sep_subbackline(sepbackmap *bkmap, int y, void *line, int dtype); 95 | int sep_backrmsline(sepbackmap *bkmap, int y, void *line, int dtype); 96 | /* Evaluate the background, RMS, or variance at line y. 97 | * Uses bicubic spline interpolation between background map verticies. 98 | * The second function subtracts the background from the input array. 99 | * Line must be an array with same width as original image. */ 100 | 101 | int sep_backarray(sepbackmap *bkmap, void *arr, int dtype); 102 | int sep_subbackarray(sepbackmap *bkmap, void *arr, int dtype); 103 | int sep_backrmsarray(sepbackmap *bkmap, void *arr, int dtype); 104 | /* Evaluate the background, RMS, or variance for entire image. 105 | * Uses bicubic spline interpolation between background map verticies. 106 | * The second function subtracts the background from the input array. 107 | * Arr must be an array of the same size as original image. */ 108 | 109 | void sep_freeback(sepbackmap *bkmap); 110 | /* Free memory associated with bkmap */ 111 | 112 | /*-------------------------- source extraction ------------------------------*/ 113 | 114 | typedef struct 115 | { 116 | float thresh; /* threshold (ADU) */ 117 | int npix; /* # pixels extracted (size of pix array) */ 118 | int tnpix; /* # pixels above thresh (unconvolved) */ 119 | int xmin,xmax,ymin,ymax; /* x,y limits */ 120 | double x, y; /* barycenter (first moments) */ 121 | double x2,y2,xy; /* second moments */ 122 | float a, b, theta; /* ellipse parameters */ 123 | float cxx,cyy,cxy; /* ellipse parameters (alternative) */ 124 | float cflux; /* total flux of pixels (convolved im) */ 125 | float flux; /* total flux of pixels (unconvolved) */ 126 | float cpeak; /* peak intensity (ADU) (convolved) */ 127 | float peak; /* peak intensity (ADU) (unconvolved) */ 128 | int xcpeak, ycpeak; /* x, y coords of peak (convolved) pixel */ 129 | int xpeak, ypeak; /* x, y coords of peak (unconvolved) pixel */ 130 | short flag; /* extraction flags */ 131 | int *pix; /* pixel array (length is npix) */ 132 | } sepobj; 133 | 134 | int sep_extract(void *image, /* image array */ 135 | void *noise, /* noise array (can be NULL) [NULL] */ 136 | int dtype, /* data type of image */ 137 | int ndtype, /* data type of noise */ 138 | short noise_flag, /* See detail below. */ 139 | int w, int h, /* width, height of image & noise */ 140 | float thresh, /* detection threshold [1.5*sigma] */ 141 | int minarea, /* minimum area in pixels [5] */ 142 | float *conv, /* convolution array (can be NULL) */ 143 | /* [{1 2 1 2 4 2 1 2 1}] */ 144 | int convw, int convh, /* w, h of convolution array [3,3] */ 145 | int deblend_nthresh, /* deblending thresholds [32] */ 146 | double deblend_cont, /* min. deblending contrast [0.005] */ 147 | int clean_flag, /* perform cleaning? [1] */ 148 | double clean_param, /* clean parameter [1.0] */ 149 | sepobj **objects, /* OUTPUT: object array */ 150 | int *nobj); /* OUTPUT: number of objects */ 151 | /* Extract sources from an image. 152 | * 153 | * Source Extractor defaults are shown in [ ] above. 154 | * 155 | * Notes 156 | * ----- 157 | * `dtype` and `ndtype` indicate the data type (float, int, double) of the 158 | * image and noise arrays, respectively. 159 | * 160 | * If `noise` is NULL, thresh is interpreted as an absolute threshold. 161 | * 162 | * If `noise` is not null, thresh is interpreted as a relative threshold 163 | * (the absolute threshold will be thresh*noise). `noise_flag` can be used 164 | * to alter this behavior. 165 | * 166 | */ 167 | 168 | /* set and get the size of the pixel stack used in extract() */ 169 | void sep_set_extract_pixstack(size_t val); 170 | size_t sep_get_extract_pixstack(void); 171 | 172 | void sep_freeobjarray(sepobj *objects, int nobj); 173 | /* free memory associated with an sepobj array, including pixel lists */ 174 | 175 | /*-------------------------- aperture photometry ----------------------------*/ 176 | 177 | int sep_sum_circle(void *data, /* data array */ 178 | void *error, /* error value or array or NULL */ 179 | void *mask, /* mask array (can be NULL) */ 180 | int dtype, /* data dtype code */ 181 | int edtype, /* error dtype code */ 182 | int mdtype, /* mask dtype code */ 183 | int w, int h, /* width, height of input arrays */ 184 | double maskthresh, /* pixel masked if mask > maskthresh */ 185 | double gain, /* (poisson counts / data unit) */ 186 | short inflags, /* input flags (see below) */ 187 | double x, /* center of aperture in x */ 188 | double y, /* center of aperture in y */ 189 | double r, /* radius of aperture */ 190 | int subpix, /* subpixel sampling */ 191 | double *sum, /* OUTPUT: sum */ 192 | double *sumerr, /* OUTPUT: error on sum */ 193 | double *area, /* OUTPUT: area included in sum */ 194 | short *flag); /* OUTPUT: flags */ 195 | /* Sum array values within a circular aperture. 196 | * 197 | * Notes 198 | * ----- 199 | * error : Can be a scalar (default), an array, or NULL 200 | * If an array, set the flag SEP_ERROR_IS_ARRAY in `inflags`. 201 | * Can represent 1-sigma std. deviation (default) or variance. 202 | * If variance, set the flag SEP_ERROR_IS_VARIANCE in `inflags`. 203 | * 204 | * gain : If 0.0, poisson noise on sum is ignored when calculating error. 205 | * Otherwise, (sum / gain) is added to the variance on sum. 206 | * 207 | * area : Total pixel area included in sum. Includes masked pixels that were 208 | * corrected. The area can differ from the exact area of a circle due 209 | * to inexact subpixel sampling and intersection with array boundaries. 210 | */ 211 | 212 | int sep_sum_circann(void *data, void *error, void *mask, 213 | int dtype, int edtype, int mdtype, int w, int h, 214 | double maskthresh, double gain, short inflags, 215 | double x, double y, double rin, double rout, int subpix, 216 | double *sum, double *sumerr, double *area, short *flag); 217 | 218 | int sep_sum_ellipse(void *data, void *error, void *mask, 219 | int dtype, int edtype, int mdtype, int w, int h, 220 | double maskthresh, double gain, short inflag, 221 | double x, double y, double a, double b, double theta, 222 | double r, int subpix, 223 | double *sum, double *sumerr, double *area, short *flag); 224 | 225 | int sep_sum_ellipann(void *data, void *error, void *mask, 226 | int dtype, int edtype, int mdtype, int w, int h, 227 | double maskthresh, double gain, short inflag, 228 | double x, double y, double a, double b, double theta, 229 | double rin, double rout, int subpix, 230 | double *sum, double *sumerr, double *area, short *flag); 231 | 232 | int sep_kron_radius(void *data, void *mask, int dtype, int mdtype, 233 | int w, int h, double maskthresh, double x, double y, 234 | double cxx, double cyy, double cxy, double r, 235 | double *kronrad, short *flag); 236 | /* Calculate Kron radius within an ellipse given by 237 | * 238 | * cxx*(x'-x)^2 + cyy*(y'-y)^2 + cxy*(x'-x)*(y'-y) < r^2 239 | * 240 | * The Kron radius is sum(r_i * v_i) / sum(v_i) where v_i is the value of pixel 241 | * i and r_i is the "radius" of pixel i, as given by the left hand side of 242 | * the above equation. 243 | * 244 | * Flags that might be set: 245 | * SEP_APER_HASMASKED - at least one of the pixels in the ellipse is masked. 246 | * SEP_APER_ALLMASKED - All pixels in the ellipse are masked. kronrad = 0. 247 | * SEP_APER_NONPOSITIVE - There was a nonpositive numerator or deminator. 248 | * kronrad = 0. 249 | */ 250 | 251 | void sep_set_ellipse(unsigned char *arr, int w, int h, 252 | double x, double y, double cxx, double cyy, double cxy, 253 | double r, unsigned char val); 254 | /* Set array elements within an ellipitcal aperture to a given value. 255 | * 256 | * Ellipse: cxx*(x'-x)^2 + cyy*(y'-y)^2 + cxy*(x'-x)*(y'-y) = r^2 257 | */ 258 | 259 | int sep_ellipse_axes(double cxx, double cyy, double cxy, 260 | double *a, double *b, double *theta); 261 | void sep_ellipse_coeffs(double a, double b, double theta, 262 | double *cxx, double *cyy, double *cxy); 263 | 264 | /*----------------------- info & error messaging ----------------------------*/ 265 | 266 | extern char *sep_version_string; 267 | /* library version string (e.g., "0.2.0") */ 268 | 269 | void sep_get_errmsg(int status, char *errtext); 270 | /* Return a short descriptive error message that corresponds to the input 271 | * error status value. The message may be up to 60 characters long, plus 272 | * the terminating null character. */ 273 | 274 | void sep_get_errdetail(char *errtext); 275 | /* Return a longer error message with more specifics about the problem. 276 | The message may be up to 512 characters */ 277 | -------------------------------------------------------------------------------- /scratch/scala/libs/jfits-0.94.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/scala/libs/jfits-0.94.jar -------------------------------------------------------------------------------- /scratch/scala/libs/libsep.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/scala/libs/libsep.a -------------------------------------------------------------------------------- /scratch/scala/test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "sep.h" 4 | 5 | int main(){ 6 | double darray[6][6]; 7 | for(int i=0; i<6; i++) 8 | for(int j=0; j<6; j++) 9 | darray[i][j] = 0.0; 10 | 11 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 12 | p->back = malloc(sizeof(float)*4); 13 | p->dback = malloc(sizeof(float)*4); 14 | p->sigma = malloc(sizeof(float)*4); 15 | p->dsigma = malloc(sizeof(float)*4); 16 | 17 | p->h = 6; 18 | p->w = 6; 19 | p->bh = 3; 20 | p->bw = 3; 21 | p->nx = 2; 22 | p->ny = 2; 23 | p->n = 4; 24 | p->globalback = 0.08958334; 25 | p->globalrms = 0.3142697; 26 | 27 | float *pp = p->back; 28 | for(int i=0; in; i++){ 29 | *(pp++) = 0.08958334; 30 | } 31 | 32 | pp = p->dback; 33 | for(int i=0; in; i++){ 34 | *(pp++) = 0.0; 35 | } 36 | 37 | pp = p->sigma; 38 | for(int i=0; in; i++){ 39 | *(pp++) = 0.3142697; 40 | } 41 | 42 | pp = p->dsigma; 43 | for(int i=0; in; i++){ 44 | *(pp++) = 0.0; 45 | } 46 | 47 | int status = sep_backarray(p, darray, 82); 48 | printf("%f, \n", darray[0][0]); 49 | } -------------------------------------------------------------------------------- /scratch/spark-ec2/README.md: -------------------------------------------------------------------------------- 1 | #Compilation instructions on EC2 2 | 3 | ## Install jfits-0.94 jar 4 | 5 | mvn install:install-file -Dfile=libs/jfits-0.94.jar -DgroupId="org.esa" -DartifactId="fits" -Dpackaging="jar" -Dversion="0.94" 6 | 7 | ## Compile the SEP library, this step can be skipped now 8 | 9 | 2.1 scons 10 | 11 | 2.2 cp src/libsep.so* /root/Kira/scratch/spark/libs/ 12 | 13 | ## Compile the JNI library, this step can be skipped now 14 | 15 | gcc Background.c Extractor.c -std=c99 -o libBackgroundImpl.so -lc -shared -I"/usr/lib/jvm/java-1.7.0/include" -I"/u 16 | sr/lib/jvm/java-1.7.0/include/linux" -L"/root/Kira/scratch/spark-ec2/libs" -lsep -I"/root/Kira/scratch/spark-ec2/libs/include" -fPIC 17 | 18 | ## Compile Kira 19 | 20 | mvn clean package 21 | 22 | ## Update spark-env.sh 23 | 24 | 5.1 edit spark-env.sh to launch mutliple JVMs each with one worker 25 | 26 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/Kira/scratch/spark-ec2/libs 27 | 28 | 5.2 edit spark-default.conf to change the memory for each worker 29 | 30 | spark.executor.memory 6144m 31 | 32 | spark.executor.extraClassPath /root/ephemeral-hdfs/conf:/root/Kira/scratch/spark-ec2/target/appassembler/repo/org/esa/fits/0.94/fits-0.94.jar 33 | 34 | spark.default.parallelism 1 35 | 36 | spark.locality.wait 0 37 | 38 | 5.2 /root/spark/sbin/stop-all.sh 39 | 40 | 5.3 /root/spark/sbin/start-all.sh 41 | 42 | ## Run Kira 43 | 44 | bin/kira-submit -------------------------------------------------------------------------------- /scratch/spark-ec2/bin/compute-kira-classpath.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Next three commands set CLASSPATH like appassembler 24 | BASEDIR="$KIRA_REPO"/target/appassembler 25 | REPO="$BASEDIR"/repo 26 | if [ ! -f "$BASEDIR"/bin/kira ]; then 27 | echo "Failed to find appassembler scripts in $BASEDIR/bin" 28 | echo "You need to build Madd before running this program" 29 | exit 1 30 | fi 31 | eval $(cat "$BASEDIR"/bin/kira | grep "^CLASSPATH") 32 | 33 | echo "$CLASSPATH" 34 | -------------------------------------------------------------------------------- /scratch/spark-ec2/bin/compute-kira-jars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | CLASSPATH=$("$KIRA_REPO"/bin/compute-kira-classpath.sh) 24 | 25 | # list of jars to ship with spark; trim off the first and last from the CLASSPATH 26 | # TODO: brittle? assumes appassembler always puts the $BASE/etc first and the CLI jar last 27 | KIRA_JARS=$(echo "$CLASSPATH" | tr ":" "," | cut -d "," -f 2- | rev | cut -d "," -f 2- | rev) 28 | 29 | echo "$KIRA_JARS" 30 | -------------------------------------------------------------------------------- /scratch/spark-ec2/bin/kira-submit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where MADD is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Get list of required jars for MADD 24 | KIRA_JARS=$("$KIRA_REPO"/bin/compute-kira-jars.sh) 25 | 26 | # Find the MADD CLI jar 27 | num_versions=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/ | wc -l) 28 | if [ "$num_versions" -eq "0" ]; then 29 | echo "Failed to find kira jar in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 30 | echo "You need to build KIRA before running this program." 31 | exit 1 32 | fi 33 | if [ "$num_versions" -gt "1" ]; then 34 | versions_list=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira) 35 | echo "Found multiple KIRA versions in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 36 | echo "$versions_list" 37 | echo "Please remove all but one." 38 | exit 1 39 | fi 40 | KIRA_JAR="$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/*/Kira*.jar)" 41 | 42 | # Find spark-submit script 43 | if [ -z "$SPARK_HOME" ]; then 44 | echo "Attempting to use 'spark-submit' on default path; you might need to set SPARK_HOME" 45 | SPARK_SUBMIT=spark-submit 46 | else 47 | SPARK_SUBMIT="$SPARK_HOME"/bin/spark-submit 48 | fi 49 | 50 | # Split args into Spark args and MADD args 51 | # NOTE: if Spark uses gatherSparkSubmitOpts in spark-submit, this is unnecessary 52 | function usage() { 53 | echo "adam-submit " 54 | exit 0 55 | } 56 | source "$SPARK_HOME"/bin/utils.sh 57 | SUBMIT_USAGE_FUNCTION=usage 58 | gatherSparkSubmitOpts "$@" 59 | 60 | # --master local-cluster[8,1,8192] \ 61 | # --jars "$KIRA_JARS" \ 62 | 63 | # submit the job to Spark 64 | "$SPARK_SUBMIT" \ 65 | --class Kira \ 66 | "${SUBMISSION_OPTS[@]}" \ 67 | "$KIRA_JAR" \ 68 | "${APPLICATION_OPTS[@]}" 69 | -------------------------------------------------------------------------------- /scratch/spark-ec2/bin/kira-submit-local-jar: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where MADD is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Get list of required jars for MADD 24 | KIRA_JARS=$("$KIRA_REPO"/bin/compute-kira-jars.sh) 25 | 26 | KIRA_JARS=`echo $KIRA_JARS|sed 's/,/,local:/g'` 27 | KIRA_JARS=local:$KIRA_JARS 28 | 29 | # Find the MADD CLI jar 30 | num_versions=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/ | wc -l) 31 | if [ "$num_versions" -eq "0" ]; then 32 | echo "Failed to find kira jar in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 33 | echo "You need to build KIRA before running this program." 34 | exit 1 35 | fi 36 | if [ "$num_versions" -gt "1" ]; then 37 | versions_list=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira) 38 | echo "Found multiple KIRA versions in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 39 | echo "$versions_list" 40 | echo "Please remove all but one." 41 | exit 1 42 | fi 43 | KIRA_JAR="$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/*/Kira*.jar)" 44 | 45 | # Find spark-submit script 46 | if [ -z "$SPARK_HOME" ]; then 47 | echo "Attempting to use 'spark-submit' on default path; you might need to set SPARK_HOME" 48 | SPARK_SUBMIT=spark-submit 49 | else 50 | SPARK_SUBMIT="$SPARK_HOME"/bin/spark-submit 51 | fi 52 | 53 | # Split args into Spark args and MADD args 54 | # NOTE: if Spark uses gatherSparkSubmitOpts in spark-submit, this is unnecessary 55 | function usage() { 56 | echo "adam-submit " 57 | exit 0 58 | } 59 | source "$SPARK_HOME"/bin/utils.sh 60 | SUBMIT_USAGE_FUNCTION=usage 61 | gatherSparkSubmitOpts "$@" 62 | 63 | # --master local-cluster[8,1,8192] \ 64 | # --jars "$KIRA_JARS" \ 65 | 66 | echo $KIRA_JARS 67 | # submit the job to Spark 68 | "$SPARK_SUBMIT" \ 69 | --class Kira \ 70 | "${SUBMISSION_OPTS[@]}" \ 71 | "local:$KIRA_JAR" \ 72 | "${APPLICATION_OPTS[@]}" 73 | -------------------------------------------------------------------------------- /scratch/spark-ec2/data/image1.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/data/image1.fits -------------------------------------------------------------------------------- /scratch/spark-ec2/data/image2.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/data/image2.fits -------------------------------------------------------------------------------- /scratch/spark-ec2/data/image3.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/data/image3.fits -------------------------------------------------------------------------------- /scratch/spark-ec2/data/image4.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/data/image4.fits -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/download-gluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ~ 4 | #download the glusterfs tar ball 5 | wget http://download.gluster.org/pub/gluster/glusterfs/LATEST/glusterfs-3.6.2.tar.gz 6 | 7 | #untar glusterfs 8 | tar -zxf glusterfs-3.6.2.tar.gz 9 | 10 | #install required packages on master 11 | yum -y install automake autoconf libtool flex bison openssl-devel libxml2-devel python-devel libaio-devel libibverbs-devel librdmacm-devel readline-devel lvm2-devel glib2-devel 12 | 13 | #install required packages on all nodes 14 | for h in `cat ~/spark/conf/slaves` 15 | do 16 | ssh $h "yum -y install automake autoconf libtool flex bison openssl-devel libxml2-devel python-devel libaio-devel libibverbs-devel librdmacm-devel readline-devel lvm2-devel glib2-devel" 17 | done 18 | 19 | #compile glusterfs on master 20 | cd glusterfs-3.6.2 21 | ./configure 22 | make 23 | make install 24 | cd .. 25 | 26 | #copy glusterfs-3.6.2 to all nodes 27 | ~/spark-ec2/copy-dir glusterfs-3.6.2 28 | 29 | #install glustefs on all nodes 30 | for h in `cat ~/spark/conf/slaves` 31 | do 32 | ssh $h "cd glusterfs-3.6.2;make install" 33 | done 34 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/ec2/run-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export LD_LIBRARY_PATH=/mnt/gluster/sep/src 3 | cd /mnt/gluster/sep/data 4 | inlist=$1 5 | indir=$2 6 | outdir=$3 7 | #inlist=tmp/x0000 8 | #outdir=$SCRATCH/output/catalog_8_8 9 | 10 | proc=8 11 | 12 | i=0 13 | 14 | for f in `cat $inlist` 15 | do 16 | base=`basename $f` 17 | #strace -T -o /dev/shm/${base}.strace /scratch1/scratchdirs/zhaozhan/sextractor-2.19.5/bin/sex $indir/$f -CATALOG_NAME $outdir/${base}.cat & 18 | /mnt/gluster/sep/ctest/test_image $indir/$f $outdir/${base}.cat & 19 | pids="$pids $!" 20 | i=$((i+1)) 21 | if [ "${i}" -eq "${proc}" ]; 22 | then 23 | wait $pids 24 | pids="" 25 | i=0 26 | fi 27 | done 28 | wait 29 | date=`date +%s.%2N` 30 | echo "finish time $date" >> log 31 | #sleep 1500 32 | 33 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/ec2/run-para.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | indir=$1 4 | outdir=$2 5 | workdir="tmp" 6 | mkdir -p $workdir 7 | #ccm_file=`sort -u $PBS_NODEFILE` 8 | #num_nodes=`sort -u $PBS_NODEFILE|wc -l` 9 | 10 | date=`date +%s.%2N` 11 | echo "start time $date" >> log 12 | 13 | ccm_file=`cat ~/spark/conf/slaves` 14 | num_nodes=`sort -u ~/spark/conf/slaves | wc -l` 15 | #num_nodes=4 16 | ls -l $indir | tail -n +2 | awk '{print $9}' > $workdir/file.list 17 | num_files=`wc -l $workdir/file.list| awk '{print $1}'` 18 | #echo $num_files 19 | num_lines=$(($num_files/$num_nodes+1)) 20 | 21 | cd $workdir 22 | split file.list -l $num_lines -d -a 4 23 | cd .. 24 | 25 | ls $workdir/x* > $workdir/part.list 26 | rm $workdir/nodes 27 | for h in $ccm_file 28 | do 29 | echo $h >> $workdir/nodes 30 | done 31 | #cp nodes $workdir/nodes 32 | 33 | date=`date +%s.%2N` 34 | echo "serial part finishes at $date" >> log 35 | 36 | paste $workdir/nodes $workdir/part.list | while read node file 37 | do 38 | #echo "ssh $node $SCRATCH/sep/data/run-local.sh $file $indir $outdir" 39 | (ssh -n $node /mnt/gluster/sep/data/run-local.sh /mnt/gluster/sep/data/$file $indir $outdir &) 40 | done 41 | #wait 42 | 43 | #sleep 1800 44 | #rm -rf $outdir/* 45 | #date=`date +%s.%2N` 46 | #echo "start time $date" >> log 47 | #paste $workdir/nodes $workdir/part.list | while read node file 48 | #do 49 | # #echo "ssh $node $SCRATCH/sep/data/run-local.sh $file $indir $outdir" 50 | # (ssh -n $node $SCRATCH/sep/data/run-local.sh $SCRATCH/sep/data/$file $indir $outdir &) 51 | #done 52 | #sleep 1800 53 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/edison/run-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $SCRATCH/sep/data 4 | inlist=$1 5 | indir=$2 6 | outdir=$3 7 | #inlist=tmp/x0000 8 | #outdir=$SCRATCH/output/catalog_8_8 9 | 10 | proc=24 11 | 12 | i=0 13 | 14 | for f in `cat $inlist` 15 | do 16 | base=`basename $f` 17 | #strace -T -o /dev/shm/${base}.strace /scratch1/scratchdirs/zhaozhan/sextractor-2.19.5/bin/sex $indir/$f -CATALOG_NAME $outdir/${base}.cat & 18 | /scratch1/scratchdirs/zhaozhan/sextractor-2.19.5/bin/sex $indir/$f -CATALOG_NAME $outdir/${base}.cat -CHECKIMAGE_TYPE NONE & 19 | pids="$pids $!" 20 | i=$((i+1)) 21 | if [ "${i}" -eq "${proc}" ]; 22 | then 23 | wait $pids 24 | pids="" 25 | i=0 26 | fi 27 | done 28 | wait 29 | date=`date +%s.%2N` 30 | echo "finish time $date" >> log 31 | #sleep 1500 32 | 33 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/edison/run-para.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | indir=$1 4 | outdir=$2 5 | workdir="tmp" 6 | mkdir -p $workdir 7 | 8 | date=`date +%s.%2N` 9 | echo "start time $date" >> log 10 | 11 | ccm_file=`sort -u $PBS_NODEFILE` 12 | num_nodes=`sort -u $PBS_NODEFILE|wc -l` 13 | #num_nodes=4 14 | ls -l $indir | tail -n +2 | awk '{print $9}' > $workdir/file.list 15 | num_files=`wc -l $workdir/file.list| awk '{print $1}'` 16 | #echo $num_files 17 | num_lines=$(($num_files/$num_nodes+1)) 18 | 19 | cd $workdir 20 | split file.list -l $num_lines -d -a 4 21 | cd .. 22 | 23 | ls $workdir/x* > $workdir/part.list 24 | rm $workdir/nodes 25 | for h in $ccm_file 26 | do 27 | echo $h >> $workdir/nodes 28 | done 29 | #cp nodes $workdir/nodes 30 | 31 | paste $workdir/nodes $workdir/part.list | while read node file 32 | do 33 | #echo "ssh $node $SCRATCH/sep/data/run-local.sh $file $indir $outdir" 34 | (ssh -n $node $SCRATCH/sep/data/run-local.sh $SCRATCH/sep/data/$file $indir $outdir &) 35 | done 36 | #wait 37 | 38 | #sleep 1800 39 | #rm -rf $outdir/* 40 | #date=`date +%s.%2N` 41 | #echo "start time $date" >> log 42 | #paste $workdir/nodes $workdir/part.list | while read node file 43 | #do 44 | # #echo "ssh $node $SCRATCH/sep/data/run-local.sh $file $indir $outdir" 45 | # (ssh -n $node $SCRATCH/sep/data/run-local.sh $SCRATCH/sep/data/$file $indir $outdir &) 46 | #done 47 | #sleep 1800 48 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/pom.xml-spark-1.2.2-SNAPSHOT: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.sonatype.oss 5 | oss-parent 6 | 7 7 | 8 | 9 | 10 | 4.0.0 11 | Kira 12 | Kira 13 | 0.0.1-SNAPSHOT 14 | jar 15 | Kira 16 | A Spark-based system for astronomical object detection 17 | 18 | 19 | 1.7 20 | 2.10.4 21 | 2.10 22 | 1.2.2-SNAPSHOT 23 | 1.7.6 24 | 1.4.3 25 | 2.2.0 26 | 0.99.2 27 | 0.5.0 28 | 29 | 30 | 31 | target/scala-${scala.version}/classes 32 | target/scala-${scala.version}/test-classes 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-assembly-plugin 38 | 2.4.1 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-javadoc-plugin 43 | 2.9.1 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-release-plugin 48 | 2.5 49 | 50 | forked-path 51 | false 52 | -Psonatype-oss-release 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-gpg-plugin 58 | 1.5 59 | 60 | 61 | net.alchim31.maven 62 | scala-maven-plugin 63 | 3.1.5 64 | 65 | 66 | org.apache.maven.plugins 67 | maven-clean-plugin 68 | 2.5 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | 3.1 74 | 75 | ${java.version} 76 | ${java.version} 77 | 78 | 79 | 80 | org.apache.maven.plugins 81 | maven-install-plugin 82 | 2.5.1 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-jar-plugin 87 | 2.4 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-resources-plugin 92 | 2.6 93 | 94 | 95 | org.codehaus.mojo 96 | appassembler-maven-plugin 97 | 1.8 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.17 103 | 104 | 105 | org.scalatest 106 | scalatest-maven-plugin 107 | 1.0-RC2 108 | 109 | 110 | org.codehaus.mojo 111 | build-helper-maven-plugin 112 | 1.7 113 | 114 | 115 | org.apache.avro 116 | avro-maven-plugin 117 | ${avro.version} 118 | 119 | 120 | 121 | 122 | 123 | org.codehaus.mojo 124 | appassembler-maven-plugin 125 | 126 | 127 | 128 | Kira 129 | kira 130 | 131 | 132 | 133 | 134 | 135 | package 136 | 137 | assemble 138 | 139 | 140 | 141 | 142 | 143 | net.alchim31.maven 144 | scala-maven-plugin 145 | 146 | 147 | scala-compile-first 148 | process-resources 149 | 150 | compile 151 | 152 | 153 | 154 | -g:vars 155 | 156 | 157 | 158 | 159 | scala-test-compile-first 160 | process-test-resources 161 | 162 | testCompile 163 | 164 | 165 | 166 | attach-scaladocs 167 | verify 168 | 169 | doc-jar 170 | 171 | 172 | 173 | 174 | ${scala.version} 175 | incremental 176 | true 177 | 178 | -unchecked 179 | -optimise 180 | -deprecation 181 | 182 | 183 | -Xms64m 184 | -Xms1024m 185 | -Xmx1024m 186 | 187 | 188 | -source 189 | ${java.version} 190 | -target 191 | ${java.version} 192 | 193 | 194 | 195 | 196 | org.scalariform 197 | scalariform-maven-plugin 198 | 0.1.4 199 | 200 | 201 | default-cli 202 | 203 | format 204 | 205 | 206 | true 207 | true 208 | true 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | org.esa 220 | fits 221 | 0.94 222 | 223 | 224 | com.esotericsoftware.kryo 225 | kryo 226 | 2.21 227 | 228 | 229 | com.twitter 230 | chill-avro 231 | ${chill.version} 232 | 233 | 234 | com.twitter 235 | chill_2.11 236 | 237 | 238 | com.twitter 239 | chill-bijection_2.11 240 | 241 | 242 | com.twitter 243 | bijection-avro_2.11 244 | 245 | 246 | 247 | 248 | com.twitter 249 | bijection-avro_${scala.artifact.suffix} 250 | 0.7.0 251 | 252 | 253 | com.twitter 254 | chill-bijection_${scala.artifact.suffix} 255 | ${chill.version} 256 | 257 | 258 | com.twitter 259 | chill_${scala.artifact.suffix} 260 | ${chill.version} 261 | 262 | 263 | org.apache.hadoop 264 | hadoop-client 265 | ${hadoop.version} 266 | 267 | 268 | asm 269 | asm 270 | 271 | 272 | org.jboss.netty 273 | netty 274 | 275 | 276 | org.codehaus.jackson 277 | * 278 | 279 | 280 | org.sonatype.sisu.inject 281 | * 282 | 283 | 284 | javax.servlet 285 | servlet-api 286 | 287 | 288 | 289 | 290 | org.apache.spark 291 | spark 292 | ${spark.version} 293 | 294 | 295 | org.apache.hadoop 296 | hadoop-client 297 | 298 | 299 | org.twitter 300 | chill_${scala.artifact.suffix} 301 | 302 | 303 | org.apache.hadoop 304 | hadoop-mapreduce 305 | 306 | 307 | 308 | 309 | org.apache.avro 310 | avro 311 | ${avro.version} 312 | 313 | 314 | com.twitter 315 | parquet-avro 316 | ${parquet.version} 317 | 318 | 319 | 320 | 321 | 322 | 323 | org.esa 324 | fits 325 | 326 | 327 | org.apache.spark 328 | spark 329 | 330 | 331 | org.apache.hadoop 332 | hadoop-client 333 | 334 | 335 | org.apache.avro 336 | avro 337 | 338 | 339 | com.twitter 340 | parquet-avro 341 | 342 | 343 | com.twitter 344 | chill-avro 345 | 346 | 347 | com.twitter 348 | bijection-avro_${scala.artifact.suffix} 349 | 350 | 351 | com.twitter 352 | chill-bijection_${scala.artifact.suffix} 353 | 354 | 355 | com.twitter 356 | chill_${scala.artifact.suffix} 357 | 358 | 359 | 360 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | proc=$1 4 | 5 | i=0 6 | pids="" 7 | for f in `ls /mnt/data/input_8_8` 8 | do 9 | ~/sep/ctest/test_image /mnt/data/input_8_8/$f /mnt/output/catalog_8_8/$f.cat & 10 | pids="$pids $!" 11 | i=$((i+1)) 12 | if [ "${i}" -eq "${proc}" ]; 13 | then 14 | wait $pids 15 | pids="" 16 | i=0 17 | fi 18 | done 19 | wait -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/setup-gluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ~ 4 | 5 | #start glusterfs daemon on local node 6 | glusterd 7 | 8 | #start glusterfs daemon on all nodes 9 | for h in `cat ~/spark/conf/slaves` 10 | do 11 | ssh $h glusterd 12 | done 13 | 14 | sleep 10 15 | #mount partition as a glusterfs brick on local node 16 | mkdir -p /srv/sdb1/brick 17 | mount /dev/xvdb /srv/sdb1 18 | echo "/dev/xvdb /srv/sdb1 xfs defaults 0 0" | sudo tee -a /etc/fstab 19 | 20 | #mount partition as a glusterfs brick on all nodes 21 | for h in `cat ~/spark/conf/slaves` 22 | do 23 | ssh $h "mkdir -p /srv/sdb1;mount /dev/xvdb /srv/sdb1;mkdir -p /srv/sdb1/brick;echo '/dev/xvdb /srv/sdb1 xfs defaults 0 0' | tee -a /etc/fstab" 24 | done 25 | 26 | sleep 10 27 | #probe all nodes 28 | for h in `cat ~/spark/conf/slaves` 29 | do 30 | gluster peer probe $h 31 | done 32 | 33 | #configure glusterfs volume 34 | #cmd="gluster volume create testvol replica 2 " 35 | #cmd="gluster volume create testvol stripe 2 " 36 | #cmd="gluster volume create testvol stripe 2 replica 2 " 37 | cmd="gluster volume create testvol " 38 | for h in `cat ~/spark/conf/slaves` 39 | do 40 | cmd="$cmd $h:/srv/sdb1/brick" 41 | done 42 | echo $cmd 43 | 44 | $cmd 45 | 46 | sleep 10 47 | #start glusterfs volume 48 | gluster volume start testvol 49 | 50 | mkdir -p /mnt/gluster 51 | target=`grep SPARK_MASTER_IP ~/spark/conf/spark-env.sh | cut -d '=' -f 2` 52 | mount -t glusterfs $target:/testvol /mnt/gluster 53 | 54 | for h in `cat ~/spark/conf/slaves` 55 | do 56 | ssh $h "mkdir -p /mnt/gluster; target=`grep SPARK_MASTER_IP ~/spark/conf/spark-env.sh | cut -d '=' -f 2`; mount -t glusterfs $target:/testvol /mnt/gluster" 57 | done 58 | -------------------------------------------------------------------------------- /scratch/spark-ec2/etc/spark-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export SPARK_LOCAL_DIRS="/mnt/spark" 4 | 5 | # Standalone cluster options 6 | export SPARK_MASTER_OPTS="" 7 | export SPARK_WORKER_INSTANCES=4 8 | export SPARK_WORKER_CORES=1 9 | 10 | export HADOOP_HOME="/root/ephemeral-hdfs" 11 | export SPARK_MASTER_IP=ec2-54-153-51-7.us-west-1.compute.amazonaws.com 12 | export MASTER=`cat /root/spark-ec2/cluster-url` 13 | 14 | export SPARK_SUBMIT_LIBRARY_PATH="$SPARK_SUBMIT_LIBRARY_PATH:/root/ephemeral-hdfs/lib/native/" 15 | export SPARK_SUBMIT_CLASSPATH="$SPARK_CLASSPATH:$SPARK_SUBMIT_CLASSPATH:/root/ephemeral-hdfs/conf" 16 | 17 | # Bind Spark's web UIs to this machine's public EC2 hostname: 18 | export SPARK_PUBLIC_DNS=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname` 19 | 20 | # Set a high ulimit for large shuffles 21 | ulimit -n 1000000 22 | 23 | #Set LD_LIBRARY_PATH 24 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/Kira/scratch/spark-ec2/libs 25 | -------------------------------------------------------------------------------- /scratch/spark-ec2/libs/jfits-0.94.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/libs/jfits-0.94.jar -------------------------------------------------------------------------------- /scratch/spark-ec2/libs/libBackgroundImpl.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/libs/libBackgroundImpl.so -------------------------------------------------------------------------------- /scratch/spark-ec2/libs/libsep.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/libs/libsep.so -------------------------------------------------------------------------------- /scratch/spark-ec2/libs/libsep.so.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/libs/libsep.so.0 -------------------------------------------------------------------------------- /scratch/spark-ec2/libs/libsep.so.0.0.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark-ec2/libs/libsep.so.0.0.0 -------------------------------------------------------------------------------- /scratch/spark-ec2/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.sonatype.oss 5 | oss-parent 6 | 7 7 | 8 | 9 | 10 | 4.0.0 11 | Kira 12 | Kira 13 | 0.0.1-SNAPSHOT 14 | jar 15 | Kira 16 | A Spark-based system for astronomical object detection 17 | 18 | 19 | 1.7 20 | 2.10.4 21 | 2.10 22 | 1.7.6 23 | 1.2.0 24 | 1.4.3 25 | 1.0.4 26 | 0.99.2 27 | 0.5.0 28 | 29 | 30 | 31 | target/scala-${scala.version}/classes 32 | target/scala-${scala.version}/test-classes 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-assembly-plugin 38 | 2.4.1 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-javadoc-plugin 43 | 2.9.1 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-release-plugin 48 | 2.5 49 | 50 | forked-path 51 | false 52 | -Psonatype-oss-release 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-gpg-plugin 58 | 1.5 59 | 60 | 61 | net.alchim31.maven 62 | scala-maven-plugin 63 | 3.1.5 64 | 65 | 66 | org.apache.maven.plugins 67 | maven-clean-plugin 68 | 2.5 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | 3.1 74 | 75 | ${java.version} 76 | ${java.version} 77 | 78 | 79 | 80 | org.apache.maven.plugins 81 | maven-install-plugin 82 | 2.5.1 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-jar-plugin 87 | 2.4 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-resources-plugin 92 | 2.6 93 | 94 | 95 | org.codehaus.mojo 96 | appassembler-maven-plugin 97 | 1.8 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.17 103 | 104 | 105 | org.scalatest 106 | scalatest-maven-plugin 107 | 1.0-RC2 108 | 109 | 110 | org.codehaus.mojo 111 | build-helper-maven-plugin 112 | 1.7 113 | 114 | 115 | org.apache.avro 116 | avro-maven-plugin 117 | ${avro.version} 118 | 119 | 120 | 121 | 122 | 123 | org.codehaus.mojo 124 | appassembler-maven-plugin 125 | 126 | 127 | 128 | Kira 129 | kira 130 | 131 | 132 | 133 | 134 | 135 | package 136 | 137 | assemble 138 | 139 | 140 | 141 | 142 | 143 | net.alchim31.maven 144 | scala-maven-plugin 145 | 146 | 147 | scala-compile-first 148 | process-resources 149 | 150 | compile 151 | 152 | 153 | 154 | -g:vars 155 | 156 | 157 | 158 | 159 | scala-test-compile-first 160 | process-test-resources 161 | 162 | testCompile 163 | 164 | 165 | 166 | attach-scaladocs 167 | verify 168 | 169 | doc-jar 170 | 171 | 172 | 173 | 174 | ${scala.version} 175 | incremental 176 | true 177 | 178 | -unchecked 179 | -optimise 180 | -deprecation 181 | 182 | 183 | -Xms64m 184 | -Xms1024m 185 | -Xmx1024m 186 | 187 | 188 | -source 189 | ${java.version} 190 | -target 191 | ${java.version} 192 | 193 | 194 | 195 | 196 | org.scalariform 197 | scalariform-maven-plugin 198 | 0.1.4 199 | 200 | 201 | default-cli 202 | 203 | format 204 | 205 | 206 | true 207 | true 208 | true 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | org.esa 220 | fits 221 | 0.94 222 | 223 | 224 | com.esotericsoftware.kryo 225 | kryo 226 | 2.21 227 | 228 | 229 | com.twitter 230 | chill-avro 231 | ${chill.version} 232 | 233 | 234 | com.twitter 235 | chill_2.11 236 | 237 | 238 | com.twitter 239 | chill-bijection_2.11 240 | 241 | 242 | com.twitter 243 | bijection-avro_2.11 244 | 245 | 246 | 247 | 248 | com.twitter 249 | bijection-avro_${scala.artifact.suffix} 250 | 0.7.0 251 | 252 | 253 | com.twitter 254 | chill-bijection_${scala.artifact.suffix} 255 | ${chill.version} 256 | 257 | 258 | com.twitter 259 | chill_${scala.artifact.suffix} 260 | ${chill.version} 261 | 262 | 263 | org.apache.hadoop 264 | hadoop-client 265 | ${hadoop.version} 266 | 267 | 268 | asm 269 | asm 270 | 271 | 272 | org.jboss.netty 273 | netty 274 | 275 | 276 | org.codehaus.jackson 277 | * 278 | 279 | 280 | org.sonatype.sisu.inject 281 | * 282 | 283 | 284 | javax.servlet 285 | servlet-api 286 | 287 | 288 | 289 | 290 | org.apache.spark 291 | spark-core_${scala.artifact.suffix} 292 | ${spark.version} 293 | 294 | 295 | org.apache.hadoop 296 | hadoop-client 297 | 298 | 299 | org.twitter 300 | chill_${scala.artifact.suffix} 301 | 302 | 303 | org.apache.hadoop 304 | hadoop-mapreduce 305 | 306 | 307 | 308 | 309 | org.apache.avro 310 | avro 311 | ${avro.version} 312 | 313 | 314 | com.twitter 315 | parquet-avro 316 | ${parquet.version} 317 | 318 | 319 | 320 | 321 | 322 | 323 | org.esa 324 | fits 325 | 326 | 327 | org.apache.spark 328 | spark-core_${scala.artifact.suffix} 329 | 330 | 331 | org.apache.hadoop 332 | hadoop-client 333 | 334 | 335 | org.apache.avro 336 | avro 337 | 338 | 339 | com.twitter 340 | parquet-avro 341 | 342 | 343 | com.twitter 344 | chill-avro 345 | 346 | 347 | com.twitter 348 | bijection-avro_${scala.artifact.suffix} 349 | 350 | 351 | com.twitter 352 | chill-bijection_${scala.artifact.suffix} 353 | 354 | 355 | com.twitter 356 | chill_${scala.artifact.suffix} 357 | 358 | 359 | 360 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/Background.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "Background.h" 6 | #include "sep.h" 7 | 8 | void sep_back_c_java(JNIEnv *env, jclass cls, jobject obj, jclass bkmapcls, jobject bkmap, sepbackmap *p){ 9 | /*set the globalback and globalrms field as return value*/ 10 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 11 | assert(id_gback != NULL); 12 | (*env)->SetFloatField(env, bkmap, id_gback, p->globalback); 13 | 14 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 15 | assert(id_rms != NULL); 16 | (*env)->SetFloatField(env, bkmap, id_rms, p->globalrms); 17 | 18 | /*set the w and h field as return value*/ 19 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 20 | (*env)->SetIntField(env, bkmap, id_w, p->w); 21 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 22 | (*env)->SetIntField(env, bkmap, id_h, p->h); 23 | 24 | /*set the bw and bh field as return value*/ 25 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 26 | (*env)->SetIntField(env, bkmap, id_bw, p->bw); 27 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 28 | (*env)->SetIntField(env, bkmap, id_bh, p->bh); 29 | 30 | /*set the nx, ny, and n field as return value*/ 31 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 32 | (*env)->SetIntField(env, bkmap, id_nx, p->nx); 33 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 34 | (*env)->SetIntField(env, bkmap, id_ny, p->ny); 35 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 36 | (*env)->SetIntField(env, bkmap, id_n, p->n); 37 | 38 | jmethodID setBackID = (*env)->GetMethodID(env, bkmapcls, "setBack", "([F)V"); 39 | jfloatArray backArray = (*env)->NewFloatArray(env, p->n); 40 | (*env)->SetFloatArrayRegion(env, backArray, 0, p->n, p->back); 41 | (*env)->CallVoidMethod(env, bkmap, setBackID, backArray); 42 | 43 | jmethodID setDbackID = (*env)->GetMethodID(env, bkmapcls, "setDback", "([F)V"); 44 | jfloatArray dbackArray = (*env)->NewFloatArray(env, p->n); 45 | (*env)->SetFloatArrayRegion(env, dbackArray, 0, p->n, p->dback); 46 | (*env)->CallVoidMethod(env, bkmap, setDbackID, dbackArray); 47 | 48 | jmethodID setSigmaID = (*env)->GetMethodID(env, bkmapcls, "setSigma", "([F)V"); 49 | jfloatArray sigmaArray = (*env)->NewFloatArray(env, p->n); 50 | (*env)->SetFloatArrayRegion(env, sigmaArray, 0, p->n, p->sigma); 51 | (*env)->CallVoidMethod(env, bkmap, setSigmaID, sigmaArray); 52 | 53 | jmethodID setDsigmaID = (*env)->GetMethodID(env, bkmapcls, "setDsigma", "([F)V"); 54 | jfloatArray dsigmaArray = (*env)->NewFloatArray(env, p->n); 55 | (*env)->SetFloatArrayRegion(env, dsigmaArray, 0, p->n, p->dsigma); 56 | (*env)->CallVoidMethod(env, bkmap, setDsigmaID, dsigmaArray); 57 | } 58 | 59 | void sep_back_java_c(JNIEnv *env, jclass bkmapcls, jobject bkmap, sepbackmap *p, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 60 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 61 | jint h = (*env)->GetIntField(env, bkmap, id_h); 62 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 63 | jint w = (*env)->GetIntField(env, bkmap, id_w); 64 | 65 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 66 | jint bh = (*env)->GetIntField(env, bkmap, id_bh); 67 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 68 | jint bw = (*env)->GetIntField(env, bkmap, id_bw); 69 | 70 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 71 | jint nx = (*env)->GetIntField(env, bkmap, id_nx); 72 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 73 | jint ny = (*env)->GetIntField(env, bkmap, id_ny); 74 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 75 | jint n = (*env)->GetIntField(env, bkmap, id_n); 76 | 77 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 78 | jfloat gback = (*env)->GetFloatField(env, bkmap, id_gback); 79 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 80 | jfloat rms = (*env)->GetFloatField(env, bkmap, id_rms); 81 | 82 | /*need to implement new array access functions here*/ 83 | jfloat *backarray = (jfloat *)(*env)->GetFloatArrayElements(env, back, NULL); 84 | jfloat *dbackarray = (jfloat *)(*env)->GetFloatArrayElements(env, dback, NULL); 85 | jfloat *sigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, sigma, NULL); 86 | jfloat *dsigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, dsigma, NULL); 87 | 88 | p->h = h; 89 | p->w = w; 90 | p->bh = bh; 91 | p->bw = bw; 92 | p->nx = nx; 93 | p->ny = ny; 94 | p->n = n; 95 | p->globalback = gback; 96 | p->globalrms = rms; 97 | 98 | jfloat *bp = p->back; 99 | jfloat *dbp = p->dback; 100 | jfloat *sp = p->sigma; 101 | jfloat *dsp = p->dsigma; 102 | 103 | memcpy(bp, backarray, sizeof(float)*(p->n)); 104 | memcpy(dbp, dbackarray, sizeof(float)*(p->n)); 105 | memcpy(sp, sigmaarray, sizeof(float)*(p->n)); 106 | memcpy(dsp, dsigmaarray, sizeof(float)*(p->n)); 107 | 108 | (*env)->ReleaseFloatArrayElements(env, back, backarray, 0); 109 | (*env)->ReleaseFloatArrayElements(env, dback, dbackarray, 0); 110 | (*env)->ReleaseFloatArrayElements(env, sigma, sigmaarray, 0); 111 | (*env)->ReleaseFloatArrayElements(env, dsigma, dsigmaarray, 0); 112 | /*free(backarray); 113 | free(dbackarray); 114 | free(sigmaarray); 115 | free(dsigmaarray);*/ 116 | } 117 | 118 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 119 | (JNIEnv *env, jobject obj, jbyteArray data, jobjectArray maskdata, jint dtype, jint mdtype, jint w, jint h, jint 120 | bw, jint bh, jdouble mthresh, jint fw, jint fh, jdouble fthresh, jobject backmap){ 121 | 122 | int len = (*env)->GetArrayLength(env, data); 123 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 124 | 125 | jboolean *mask; 126 | if(maskdata == NULL){ 127 | mask = NULL; 128 | } 129 | else{ 130 | /*parse the mask matrix*/ 131 | int mlen1 = (*env)->GetArrayLength(env, maskdata); 132 | jbooleanArray dim = (jbooleanArray)(*env)->GetObjectArrayElement(env, maskdata, 0); 133 | int mlen2 = (*env)->GetArrayLength(env, dim); 134 | printf("mlen2: %d\n", mlen2); 135 | mask = malloc(sizeof(jboolean)* mlen1 * mlen2); 136 | for(int i=0; iGetObjectArrayElement(env, maskdata, i); 138 | jboolean *element=(*env)->GetBooleanArrayElements(env, oneDim, 0); 139 | memcpy(mask+i*mlen2, element, sizeof(jboolean)*mlen2); 140 | free(element); 141 | (*env)->ReleaseBooleanArrayElements(env, oneDim, element, 0); 142 | } 143 | jboolean *bp = (*env)->GetBooleanArrayElements(env, dim, 0); 144 | (*env)->ReleaseBooleanArrayElements(env, dim, bp, 0); 145 | } 146 | 147 | 148 | /*printf("%d\n", len); 149 | for(int i=0; iw = w; 155 | p->h = h; 156 | p->bw = bw; 157 | p->bh = bh; 158 | 159 | int status = sep_makeback(array, mask, dtype, mdtype, w, h, bw, bh, mthresh, fw, fh, fthresh, &p); 160 | 161 | //printf("%d\n", status); 162 | //printf("C sep_makeback: %d\t%d\t%f\t%f\t%d\t%d\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh, p->nx, p->ny); 163 | //printf("C sep_makeback back: %f\t dback: %f\n", *(p->back), *(p->dback)); 164 | 165 | /*Finding the sepbackmap field in the Background class*/ 166 | jclass cls = (*env)->GetObjectClass(env, obj); 167 | assert(cls != NULL); 168 | jfieldID id_bkmap = (*env)->GetFieldID(env, cls, "bkgmap", "LSepbackmap;"); 169 | assert(id_bkmap != NULL); 170 | jobject bkmap = (*env)->GetObjectField(env, obj, id_bkmap); 171 | assert(bkmap != NULL); 172 | jclass bkmapcls = (*env)->GetObjectClass(env, bkmap); 173 | assert(bkmapcls != NULL); 174 | 175 | sep_back_c_java(env, cls, obj, bkmapcls, bkmap, p); 176 | 177 | /*float *fp = p->back; 178 | for(int i=0; in; i++){ 179 | printf("back: %f\t", *(fp++)); 180 | } 181 | printf("\n"); 182 | 183 | fp = p->dback; 184 | for(int i=0; in; i++){ 185 | printf("dback: %f\t", *(fp++)); 186 | } 187 | printf("\n"); 188 | 189 | fp = p->sigma; 190 | for(int i=0; in; i++){ 191 | printf("sigma: %f\t", *(fp++)); 192 | } 193 | printf("\n"); 194 | 195 | fp = p->dsigma; 196 | for(int i=0; in; i++){ 197 | printf("dsigma: %f\t", *(fp++)); 198 | } 199 | printf("\n");*/ 200 | 201 | (*env)->ReleaseByteArrayElements(env, data, array, 0); 202 | //free(array); 203 | //printf("free p\n"); 204 | //free(p); 205 | 206 | //free mask is not right, needs to fix 207 | if(mask != NULL){ 208 | free(mask); 209 | } 210 | /*if(dim != NULL){ 211 | free(dim); 212 | } 213 | if(oneDim != NULL){ 214 | free(oneDim); 215 | }*/ 216 | 217 | return status; 218 | } 219 | 220 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 221 | (JNIEnv *env, jobject obj, jobject bkmap, jbyteArray data, jint dtype, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 222 | 223 | int len = (*env)->GetArrayLength(env, data); 224 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 225 | 226 | jclass cls = (*env)->GetObjectClass(env, bkmap); 227 | assert(cls != NULL); 228 | 229 | int backLen = (*env)->GetArrayLength(env, back); 230 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 231 | p->back = malloc(sizeof(float)*backLen); 232 | p->dback = malloc(sizeof(float)*backLen); 233 | p->sigma = malloc(sizeof(float)*backLen); 234 | p->dsigma = malloc(sizeof(float)*backLen); 235 | 236 | sep_back_java_c(env, cls, bkmap, p, back, dback, sigma, dsigma); 237 | 238 | /*printf("C sep_backarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 239 | printf("C sep_backarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 240 | printf("C sep_backarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 241 | printf("C sep_backarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma));*/ 242 | 243 | 244 | int status = sep_backarray(p, array, dtype); 245 | //printf("status: %d\n", status); 246 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 247 | 248 | //printf("%d, \n", array[0]); 249 | 250 | /*for(int i=0; iGetArrayLength(env, data); 266 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 267 | 268 | jclass cls = (*env)->GetObjectClass(env, bkgmap); 269 | assert(cls != NULL); 270 | 271 | int backLen = (*env)->GetArrayLength(env, back); 272 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 273 | p->back = malloc(sizeof(float)*backLen); 274 | p->dback = malloc(sizeof(float)*backLen); 275 | p->sigma = malloc(sizeof(float)*backLen); 276 | p->dsigma = malloc(sizeof(float)*backLen); 277 | 278 | sep_back_java_c(env, cls, bkgmap, p, back, dback, sigma, dsigma); 279 | 280 | //printf("C sep_subbackarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 281 | //printf("C sep_subbackarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 282 | //printf("C sep_subbackarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 283 | //printf("C sep_subbackarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma)); 284 | 285 | int status = sep_subbackarray(p, array, dtype); 286 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 287 | 288 | /*printf("C sep_subbackarray: result\n"); 289 | double *dp = (double *)array; 290 | for(int i=0; iReleaseByteArrayElements(env, data, array, 0); 298 | //printf("C subbackarray: free p-> *\n"); 299 | free(p->back); 300 | free(p->dback); 301 | free(p->sigma); 302 | free(p->dsigma); 303 | //free(p); 304 | return status; 305 | } 306 | 307 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/Background.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background */ 4 | 5 | #ifndef _Included_Background 6 | #define _Included_Background 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Background 12 | * Method: sep_makeback 13 | * Signature: ([B[[ZIIIIIIDIIDLBackground/Sepbackmap;)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 16 | (JNIEnv *, jobject, jbyteArray, jobjectArray, jint, jint, jint, jint, jint, jint, jdouble, jint, jint, jdouble, jobject); 17 | 18 | /* 19 | * Class: Background 20 | * Method: sep_backarray 21 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 24 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 25 | 26 | /* 27 | * Class: Background 28 | * Method: sep_backrmsarray 29 | * Signature: (LBackground/Sepbackmap;[Ljava/lang/Object;I)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Background_sep_1backrmsarray 32 | (JNIEnv *, jobject, jobject, jobjectArray, jint); 33 | 34 | /* 35 | * Class: Background 36 | * Method: sep_subbackarray 37 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Background_sep_1subbackarray 40 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 41 | 42 | /* 43 | * Class: Background 44 | * Method: sep_freeback 45 | * Signature: (LBackground/Sepbackmap;)V 46 | */ 47 | JNIEXPORT void JNICALL Java_Background_sep_1freeback 48 | (JNIEnv *, jobject, jobject); 49 | 50 | /* 51 | * Class: Background 52 | * Method: sep_extract 53 | * Signature: ([B[BIISIIFI[BIIIDZD[LBackground/Sepobj;I)I 54 | */ 55 | JNIEXPORT jint JNICALL Java_Background_sep_1extract 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 57 | 58 | /* 59 | * Class: Background 60 | * Method: sep_freeobjarray 61 | * Signature: ([LBackground/Sepobj;I)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Background_sep_1freeobjarray 64 | (JNIEnv *, jobject, jobjectArray, jint); 65 | 66 | /* 67 | * Class: Background 68 | * Method: sep_sum_circle 69 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 70 | */ 71 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circle 72 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 73 | 74 | /* 75 | * Class: Background 76 | * Method: sep_sum_circann 77 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 78 | */ 79 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circann 80 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 81 | 82 | /* 83 | * Class: Background 84 | * Method: sep_sum_ellipse 85 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 86 | */ 87 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipse 88 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 89 | 90 | /* 91 | * Class: Background 92 | * Method: sep_sum_ellipann 93 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 94 | */ 95 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipann 96 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 97 | 98 | /* 99 | * Class: Background 100 | * Method: sep_kron_radius 101 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 102 | */ 103 | JNIEXPORT void JNICALL Java_Background_sep_1kron_1radius 104 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 105 | 106 | /* 107 | * Class: Background 108 | * Method: sep_ellipse_axes 109 | * Signature: ([D[D[D[D[D[D)V 110 | */ 111 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1axes 112 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 113 | 114 | /* 115 | * Class: Background 116 | * Method: sep_ellipse_coeffs 117 | * Signature: ([D[D[D[D[D[D)V 118 | */ 119 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1coeffs 120 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 121 | 122 | /* 123 | * Class: Background 124 | * Method: sep_set_ellipse 125 | * Signature: (Ljava/lang/String;IIDDDDDDS)V 126 | */ 127 | JNIEXPORT void JNICALL Java_Background_sep_1set_1ellipse 128 | (JNIEnv *, jobject, jstring, jint, jint, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jshort); 129 | 130 | /* 131 | * Class: Background 132 | * Method: sep_set_extract_pixstack 133 | * Signature: (I)V 134 | */ 135 | JNIEXPORT void JNICALL Java_Background_sep_1set_1extract_1pixstack 136 | (JNIEnv *, jobject, jint); 137 | 138 | /* 139 | * Class: Background 140 | * Method: sep_get_extract_pixstack 141 | * Signature: ()I 142 | */ 143 | JNIEXPORT jint JNICALL Java_Background_sep_1get_1extract_1pixstack 144 | (JNIEnv *, jobject); 145 | 146 | /* 147 | * Class: Background 148 | * Method: sep_get_errmsg 149 | * Signature: (ILjava/lang/String;)V 150 | */ 151 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errmsg 152 | (JNIEnv *, jobject, jint, jstring); 153 | 154 | /* 155 | * Class: Background 156 | * Method: sep_get_errdetail 157 | * Signature: (Ljava/lang/String;)V 158 | */ 159 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errdetail 160 | (JNIEnv *, jobject, jstring); 161 | 162 | #ifdef __cplusplus 163 | } 164 | #endif 165 | #endif 166 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/Background_Sepbackmap.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepbackmap */ 4 | 5 | #ifndef _Included_Background_Sepbackmap 6 | #define _Included_Background_Sepbackmap 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/Background_Sepobj.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepobj */ 4 | 5 | #ifndef _Included_Background_Sepobj 6 | #define _Included_Background_Sepobj 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/Extractor.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Extractor */ 4 | 5 | #ifndef _Included_Extractor 6 | #define _Included_Extractor 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Extractor 12 | * Method: sep_sum_circle 13 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circle 16 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 17 | 18 | /* 19 | * Class: Extractor 20 | * Method: sep_sum_circann 21 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circann 24 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 25 | 26 | /* 27 | * Class: Extractor 28 | * Method: sep_sum_ellipse 29 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipse 32 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 33 | 34 | /* 35 | * Class: Extractor 36 | * Method: sep_sum_ellipann 37 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipann 40 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 41 | 42 | /* 43 | * Class: Extractor 44 | * Method: sep_extract 45 | * Signature: ([B[BIISIIFI[BIIIDZD[LSepobj;I)I 46 | */ 47 | JNIEXPORT jint JNICALL Java_Extractor_sep_1extract 48 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 49 | 50 | /* 51 | * Class: Extractor 52 | * Method: sep_kron_radius 53 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 54 | */ 55 | JNIEXPORT void JNICALL Java_Extractor_sep_1kron_1radius 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 57 | 58 | /* 59 | * Class: Extractor 60 | * Method: sep_ellipse_coeffs 61 | * Signature: ([D[D[D[D[D[D)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1coeffs 64 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 65 | 66 | /* 67 | * Class: Extractor 68 | * Method: sep_ellipse_axes 69 | * Signature: ([D[D[D[D[D[D)V 70 | */ 71 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1axes 72 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 73 | 74 | /* 75 | * Class: Extractor 76 | * Method: sep_set_ellipse 77 | * Signature: ([BII[D[D[D[D[D[DB)V 78 | */ 79 | JNIEXPORT void JNICALL Java_Extractor_sep_1set_1ellipse 80 | (JNIEnv *, jobject, jbyteArray, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jbyte); 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | #endif 86 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/README.md: -------------------------------------------------------------------------------- 1 | On EC2 2 | gcc Background.c Extractor.c -std=c99 -o libBackgroundImpl.so -lc -shared -I"/usr/lib/jvm/java-1.7.0/include" -I"/usr/lib/jvm/java-1.7.0/include/linux" -L"/root/Kira/scratch/spark-ec2/libs" -lsep -I"/root/Kira/scratch/spark-ec2/libs/include" -fPIC 3 | 4 | On NERSC Edison 5 | gcc Background.c Extractor.c -std=c99 -o libBackgroundImpl.so -lc -shared -I"/usr/lib64/jvm/java-1.7.0-ibm/include" -I"/usr/lib64/jvm/java-1.7.0-ibm/include/linux" -L"$SCRATCH/Kira/scratch/spark-ec2/libs" -lsep -I"$SCRATCH/Kira/scratch/spark-ec2/libs/include" -fPIC 6 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/C/test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "sep.h" 4 | 5 | int main(){ 6 | double darray[6][6]; 7 | for(int i=0; i<6; i++) 8 | for(int j=0; j<6; j++) 9 | darray[i][j] = 0.0; 10 | 11 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 12 | p->back = malloc(sizeof(float)*4); 13 | p->dback = malloc(sizeof(float)*4); 14 | p->sigma = malloc(sizeof(float)*4); 15 | p->dsigma = malloc(sizeof(float)*4); 16 | 17 | p->h = 6; 18 | p->w = 6; 19 | p->bh = 3; 20 | p->bw = 3; 21 | p->nx = 2; 22 | p->ny = 2; 23 | p->n = 4; 24 | p->globalback = 0.08958334; 25 | p->globalrms = 0.3142697; 26 | 27 | float *pp = p->back; 28 | for(int i=0; in; i++){ 29 | *(pp++) = 0.08958334; 30 | } 31 | 32 | pp = p->dback; 33 | for(int i=0; in; i++){ 34 | *(pp++) = 0.0; 35 | } 36 | 37 | pp = p->sigma; 38 | for(int i=0; in; i++){ 39 | *(pp++) = 0.3142697; 40 | } 41 | 42 | pp = p->dsigma; 43 | for(int i=0; in; i++){ 44 | *(pp++) = 0.0; 45 | } 46 | 47 | int status = sep_backarray(p, darray, 82); 48 | printf("%f, \n", darray[0][0]); 49 | } -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/scala/Kira.scala: -------------------------------------------------------------------------------- 1 | import org.apache.spark.SparkContext 2 | import org.apache.spark.SparkContext._ 3 | import org.apache.spark.SparkConf 4 | import java.io._ 5 | import java.util.zip.CRC32 6 | import java.util.zip.Checksum 7 | 8 | object Kira { 9 | def main(args: Array[String]) { 10 | val conf = new SparkConf().setAppName("Kira") 11 | val sc = new SparkContext(conf) 12 | 13 | val src = args(0) 14 | val flist = sc.binaryFiles(src) 15 | //val rflist = flist.repartition(384) 16 | //rflist.count 17 | //println("starting time: " + System.currentTimeMillis) 18 | val results = flist.map(c => extract_str(c._2.toArray)) 19 | val flatresults = results.flatMap(p => p.map(r => (r._1, r._2, r._3, r._4, r._5, r._6, r._7, r._8, r._9))) 20 | flatresults.saveAsTextFile(args(1)) 21 | //println("ending time: " + System.currentTimeMillis) 22 | /*val flist = new File(src).listFiles.filter(_.getName.endsWith(".fit")) 23 | 24 | val dflist = sc.parallelize(flist) 25 | val results = dflist.map(f => extract(f.toString)) 26 | 27 | val flatresults = results.flatMap(p => p.map(r => (r._1, r._2, r._3, r._4, r._5, r._6, r._7, r._8, r._9))) 28 | //println("count: " + flatresults.count) 29 | flatresults.saveAsTextFile(args(1))*/ 30 | } 31 | def extract(path: String): Array[(Int, Double, Double, Double, Double, Double, Double, Double, Short)] = { 32 | var matrix = Utils.load(path) 33 | var bkg = new Background(matrix) 34 | matrix = bkg.subfrom(matrix) 35 | val ex = new Extractor 36 | val objects = ex.extract(matrix, (1.5 * bkg.bkgmap.globalrms).toFloat) 37 | //return objects.length 38 | var x: Array[Double] = Array.ofDim[Double](objects.length) 39 | var y: Array[Double] = Array.ofDim[Double](objects.length) 40 | var a: Array[Double] = Array.ofDim[Double](objects.length) 41 | var b: Array[Double] = Array.ofDim[Double](objects.length) 42 | var theta: Array[Double] = Array.ofDim[Double](objects.length) 43 | var r: Array[Double] = Array.fill(objects.length) { 6.0 } 44 | for (i <- (0 until objects.length)) { 45 | x(i) = objects(i).x 46 | y(i) = objects(i).y 47 | a(i) = objects(i).a 48 | b(i) = objects(i).b 49 | theta(i) = objects(i).theta 50 | } 51 | 52 | var err: Array[Array[Double]] = Array.fill(matrix.length, matrix(0).length) { bkg.bkgmap.globalrms } 53 | 54 | val (flux, fluxerr, flag) = ex.sum_circle(matrix, x, y, 5.0, err = err) 55 | val (kr, flag2) = ex.kron_radius(matrix, x, y, a, b, theta, r) 56 | val kr_ellipse = kr.map(x => 2.5 * x) 57 | 58 | val (flux_auto, flux_auto_err, auto_flag) = ex.sum_ellipse(matrix, x, y, a, b, theta, kr_ellipse, err = err, subpix = 1) 59 | 60 | val retArray = (0 until objects.length).map(i => (i, x(i), y(i), flux(i), fluxerr(i), kr(i), flux_auto(i), flux_auto_err(i), auto_flag(i))).toArray 61 | 62 | return retArray 63 | } 64 | 65 | def extract_str(content: Array[Byte]): Array[(Int, Double, Double, Double, Double, Double, Double, Double, Short)] = { 66 | var matrix = Utils.load_byte(content) 67 | var bkg = new Background(matrix) 68 | matrix = bkg.subfrom(matrix) 69 | val ex = new Extractor 70 | val objects = ex.extract(matrix, (1.5 * bkg.bkgmap.globalrms).toFloat) 71 | //return objects.length 72 | var x: Array[Double] = Array.ofDim[Double](objects.length) 73 | var y: Array[Double] = Array.ofDim[Double](objects.length) 74 | var a: Array[Double] = Array.ofDim[Double](objects.length) 75 | var b: Array[Double] = Array.ofDim[Double](objects.length) 76 | var theta: Array[Double] = Array.ofDim[Double](objects.length) 77 | var r: Array[Double] = Array.fill(objects.length) { 6.0 } 78 | for (i <- (0 until objects.length)) { 79 | x(i) = objects(i).x 80 | y(i) = objects(i).y 81 | a(i) = objects(i).a 82 | b(i) = objects(i).b 83 | theta(i) = objects(i).theta 84 | } 85 | 86 | var err: Array[Array[Double]] = Array.fill(matrix.length, matrix(0).length) { bkg.bkgmap.globalrms } 87 | 88 | val (flux, fluxerr, flag) = ex.sum_circle(matrix, x, y, 5.0, err = err) 89 | 90 | val (kr, flag2) = ex.kron_radius(matrix, x, y, a, b, theta, r) 91 | val kr_ellipse = kr.map(x => 2.5 * x) 92 | 93 | val (flux_auto, flux_auto_err, auto_flag) = ex.sum_ellipse(matrix, x, y, a, b, theta, kr_ellipse, err = err, subpix = 1) 94 | 95 | val retArray = (0 until objects.length).map(i => (i, x(i), y(i), flux(i), fluxerr(i), kr(i), flux_auto(i), flux_auto_err(i), auto_flag(i))).toArray 96 | return retArray 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/scala/Sepbackmap.scala: -------------------------------------------------------------------------------- 1 | class Sepbackmap(var w: Int, var h: Int, var globalback: Float, var globalrms: Float, var bw: Int, 2 | var bh: Int, var nx: Int, var ny: Int, var n: Int, var back: Array[Float], var dback: Array[Float], 3 | var sigma: Array[Float], var dsigma: Array[Float]) { 4 | back = new Array[Float](n) 5 | dback = new Array[Float](n) 6 | sigma = new Array[Float](n) 7 | dsigma = new Array[Float](n) 8 | 9 | def this() { 10 | this(0, 0, 0.0F, 0.0F, 0, 0, 0, 0, 0, null, null, null, null) 11 | } 12 | 13 | def setBack(vback: Array[Float]) { 14 | back = new Array[Float](vback.length) 15 | for (i <- (0 until back.length)) { 16 | back(i) = vback(i) 17 | } 18 | } 19 | 20 | def setDback(vdback: Array[Float]) { 21 | dback = new Array[Float](vdback.length) 22 | for (i <- (0 until dback.length)) { 23 | dback(i) = vdback(i) 24 | } 25 | } 26 | 27 | def setSigma(vsigma: Array[Float]) { 28 | sigma = new Array[Float](vsigma.length) 29 | for (i <- (0 until sigma.length)) { 30 | sigma(i) = vsigma(i) 31 | } 32 | } 33 | 34 | def setDsigma(vdsigma: Array[Float]) { 35 | dsigma = new Array[Float](vdsigma.length) 36 | for (i <- (0 until dsigma.length)) { 37 | dsigma(i) = vdsigma(i) 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/scala/Sepobj.scala: -------------------------------------------------------------------------------- 1 | 2 | class Sepobj(var thresh: Double, var npix: Int, var tnpix: Int, var xmin: Int, var xmax: Int, 3 | var ymin: Int, var ymax: Int, var x: Double, var y: Double, var x2: Double, var y2: Double, 4 | var xy: Double, var a: Float, var b: Float, var theta: Float, var cxx: Float, var cyy: Float, 5 | var cxy: Float, var cflux: Float, var flux: Float, var cpeak: Float, var peak: Float, 6 | var xpeak: Float, var ypeak: Float, var xcpeak: Float, var ycpeak: Float, var flag: Short, var pix: Int) -------------------------------------------------------------------------------- /scratch/spark-ec2/src/main/scala/Utils.scala: -------------------------------------------------------------------------------- 1 | import java.lang.{ Double => jDouble } 2 | import java.nio.ByteBuffer 3 | import java.io._ 4 | import org.eso.fits._ 5 | 6 | object Utils { 7 | /**Helper function that converts 2d double matrix to 1d byte stream*/ 8 | def flatten(matrix: Array[Array[Double]]): Array[Byte] = { 9 | val h = matrix.length 10 | val w = matrix(0).length 11 | var stream = new Array[Byte](w * h * 8) 12 | 13 | for (i <- (0 until h)) { 14 | for (j <- (0 until w)) { 15 | var l: Long = jDouble.doubleToLongBits(matrix(i)(j)) 16 | for (k <- (0 until 8)) { 17 | stream((i * w + j) * 8 + k) = ((l >>> (k * 8)) & 0xff).toByte 18 | } 19 | } 20 | } 21 | stream 22 | } 23 | 24 | /**Helper function that converts 1d byte stream to 2d double matrix*/ 25 | def deflatten(data: Array[Byte], h: Int, w: Int): Array[Array[Double]] = { 26 | var matrix = Array.ofDim[Double](h, w) 27 | for (i <- (0 until h)) { 28 | for (j <- (0 until w)) { 29 | var bytes = new Array[Byte](8) 30 | for (k <- (0 until 8)) { 31 | bytes(7 - k) = (data((i * w + j) * 8 + k) & 0xff).toByte 32 | } 33 | matrix(i)(j) = ByteBuffer.wrap(bytes).getDouble() 34 | } 35 | } 36 | return matrix 37 | } 38 | 39 | def load(path: String): Array[Array[Double]] = { 40 | val file: FitsFile = new FitsFile(path) 41 | val hdu: FitsHDUnit = file.getHDUnit(0) 42 | val dm: FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 43 | val naxis: Array[Int] = dm.getNaxis() 44 | val ncol = naxis(0) 45 | val nval = dm.getNoValues() 46 | val nrow = nval / ncol 47 | 48 | // build and populate an array 49 | var matrix = Array.ofDim[Float](nrow, ncol) 50 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 51 | 52 | println("matrix: height: " + nrow + "\twidth: " + ncol) 53 | 54 | var rmatrix = Array.ofDim[Double](nrow, ncol) 55 | for (i <- (0 until nrow)) { 56 | for (j <- (0 until ncol)) { 57 | rmatrix(i)(j) = matrix(i)(j).toDouble 58 | } 59 | } 60 | 61 | return rmatrix 62 | } 63 | 64 | def load_byte(content: Array[Byte]): Array[Array[Double]] = { 65 | val is = new ByteArrayInputStream(content) 66 | val dis = new DataInputStream(is) 67 | val file = new FitsFile(dis, true) 68 | val hdu: FitsHDUnit = file.getHDUnit(0) 69 | val dm: FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 70 | val naxis: Array[Int] = dm.getNaxis() 71 | val ncol = naxis(0) 72 | val nval = dm.getNoValues() 73 | val nrow = nval / ncol 74 | println("nrow: " + nrow + "\t ncol:" + ncol) 75 | // build and populate an array 76 | var matrix = Array.ofDim[Float](nrow, ncol) 77 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 78 | 79 | println("matrix: height: " + nrow + "\twidth: " + ncol) 80 | 81 | var rmatrix = Array.ofDim[Double](nrow, ncol) 82 | for (i <- (0 until nrow)) { 83 | for (j <- (0 until ncol)) { 84 | rmatrix(i)(j) = matrix(i)(j).toDouble 85 | } 86 | } 87 | 88 | return rmatrix 89 | } 90 | } -------------------------------------------------------------------------------- /scratch/spark/README.md: -------------------------------------------------------------------------------- 1 | Compilation instructions on MAC 2 | 1. Install jfits-0.94.jar 3 | mvn install:install-file -Dfile=libs/jfits-0.94.jar -DgroupId="org.esa" -DartifactId="fits" -Dpackaging="jar" -Dversion="0.94" 4 | 5 | 2. Build Kira 6 | mvn clean package 7 | 8 | 3. Run Kira 9 | bin/kira-submit 10 | 11 | 4. Expected Output: 12 | Spark: 272 objects were detected 13 | 14 | 15 | Compilation instructions on EC2 16 | Install jfits-0.94 jar 17 | mvn install:install-file -Dfile=libs/jfits-0.94.jar -DgroupId="org.esa" -DartifactId="fits" -Dpackaging="jar" -Dversion="0.94" 18 | 19 | Compile the SEP library 20 | 1. scons 21 | 2. cp src/libsep.so* /root/Kira/scratch/spark/libs/ 22 | 23 | Add LD_LIBRARY_PATH to spark-env.sh 24 | export LD_LIBRARY_PATH=LD_LIBRARY_PATH=/root/Kira/scratch/spark/src/main/scala:/root/Kira/scratch/spark/libs:$LD_LIBRARY_PATH 25 | 26 | Change spark-env.sh parameter to launch multiple JVMs instead of multiple threads 27 | 28 | Compile the JNI library 29 | gcc Background.c Extractor.c -std=c99 -o libBackgroundImpl.so -lc -shared -I"/usr/lib/jvm/java-1.7.0/include" -I"/usr/lib/jvm/java-1.7.0/include/linux" -L"/root/Kira/scratch/spark/libs" -lsep -I"/root/Kira/scratch/spark/libs/include" -fPIC 30 | 31 | Compile Kira 32 | mvn clean package 33 | 34 | Run Kira 35 | bin/kira-submit -------------------------------------------------------------------------------- /scratch/spark/bin/compute-kira-classpath.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Next three commands set CLASSPATH like appassembler 24 | BASEDIR="$KIRA_REPO"/target/appassembler 25 | REPO="$BASEDIR"/repo 26 | if [ ! -f "$BASEDIR"/bin/kira ]; then 27 | echo "Failed to find appassembler scripts in $BASEDIR/bin" 28 | echo "You need to build Madd before running this program" 29 | exit 1 30 | fi 31 | eval $(cat "$BASEDIR"/bin/kira | grep "^CLASSPATH") 32 | 33 | echo "$CLASSPATH" 34 | -------------------------------------------------------------------------------- /scratch/spark/bin/compute-kira-jars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where ADAM is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | CLASSPATH=$("$KIRA_REPO"/bin/compute-kira-classpath.sh) 24 | 25 | # list of jars to ship with spark; trim off the first and last from the CLASSPATH 26 | # TODO: brittle? assumes appassembler always puts the $BASE/etc first and the CLI jar last 27 | KIRA_JARS=$(echo "$CLASSPATH" | tr ":" "," | cut -d "," -f 2- | rev | cut -d "," -f 2- | rev) 28 | 29 | echo "$KIRA_JARS" 30 | -------------------------------------------------------------------------------- /scratch/spark/bin/kira-submit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to Big Data Genomics (BDG) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The BDG licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Figure out where MADD is installed 21 | KIRA_REPO="$(cd `dirname $0`/..; pwd)" 22 | 23 | # Get list of required jars for MADD 24 | KIRA_JARS=$("$KIRA_REPO"/bin/compute-kira-jars.sh) 25 | 26 | # Find the MADD CLI jar 27 | num_versions=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/ | wc -l) 28 | if [ "$num_versions" -eq "0" ]; then 29 | echo "Failed to find kira jar in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 30 | echo "You need to build KIRA before running this program." 31 | exit 1 32 | fi 33 | if [ "$num_versions" -gt "1" ]; then 34 | versions_list=$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira) 35 | echo "Found multiple KIRA versions in $KIRA_REPO/target/appassembler/repo/Kira/Kira" 36 | echo "$versions_list" 37 | echo "Please remove all but one." 38 | exit 1 39 | fi 40 | KIRA_JAR="$(ls "$KIRA_REPO"/target/appassembler/repo/Kira/Kira/*/Kira*.jar)" 41 | 42 | # Find spark-submit script 43 | if [ -z "$SPARK_HOME" ]; then 44 | echo "Attempting to use 'spark-submit' on default path; you might need to set SPARK_HOME" 45 | SPARK_SUBMIT=spark-submit 46 | else 47 | SPARK_SUBMIT="$SPARK_HOME"/bin/spark-submit 48 | fi 49 | 50 | # Split args into Spark args and MADD args 51 | # NOTE: if Spark uses gatherSparkSubmitOpts in spark-submit, this is unnecessary 52 | function usage() { 53 | echo "adam-submit " 54 | exit 0 55 | } 56 | source "$SPARK_HOME"/bin/utils.sh 57 | SUBMIT_USAGE_FUNCTION=usage 58 | gatherSparkSubmitOpts "$@" 59 | 60 | # submit the job to Spark 61 | "$SPARK_SUBMIT" \ 62 | --class Kira \ 63 | --master local-cluster[2,1,1024] \ 64 | --jars "$KIRA_JARS" \ 65 | "${SUBMISSION_OPTS[@]}" \ 66 | "$KIRA_JAR" \ 67 | "${APPLICATION_OPTS[@]}" 68 | -------------------------------------------------------------------------------- /scratch/spark/data/image1.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/data/image1.fits -------------------------------------------------------------------------------- /scratch/spark/data/image2.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/data/image2.fits -------------------------------------------------------------------------------- /scratch/spark/data/image3.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/data/image3.fits -------------------------------------------------------------------------------- /scratch/spark/data/image4.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/data/image4.fits -------------------------------------------------------------------------------- /scratch/spark/libs/jfits-0.94.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/libs/jfits-0.94.jar -------------------------------------------------------------------------------- /scratch/spark/libs/libsep.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/libs/libsep.a -------------------------------------------------------------------------------- /scratch/spark/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.sonatype.oss 5 | oss-parent 6 | 7 7 | 8 | 9 | 10 | 4.0.0 11 | Kira 12 | Kira 13 | 0.0.1-SNAPSHOT 14 | jar 15 | Kira 16 | A Spark-based system for astronomical object detection 17 | 18 | 19 | 1.7 20 | 2.10.4 21 | 2.10 22 | 1.7.6 23 | 1.2.0 24 | 1.4.3 25 | 2.2.0 26 | 0.99.2 27 | 0.5.0 28 | 29 | 30 | 31 | target/scala-${scala.version}/classes 32 | target/scala-${scala.version}/test-classes 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-assembly-plugin 38 | 2.4.1 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-javadoc-plugin 43 | 2.9.1 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-release-plugin 48 | 2.5 49 | 50 | forked-path 51 | false 52 | -Psonatype-oss-release 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-gpg-plugin 58 | 1.5 59 | 60 | 61 | net.alchim31.maven 62 | scala-maven-plugin 63 | 3.1.5 64 | 65 | 66 | org.apache.maven.plugins 67 | maven-clean-plugin 68 | 2.5 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | 3.1 74 | 75 | ${java.version} 76 | ${java.version} 77 | 78 | 79 | 80 | org.apache.maven.plugins 81 | maven-install-plugin 82 | 2.5.1 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-jar-plugin 87 | 2.4 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-resources-plugin 92 | 2.6 93 | 94 | 95 | org.codehaus.mojo 96 | appassembler-maven-plugin 97 | 1.8 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.17 103 | 104 | 105 | org.scalatest 106 | scalatest-maven-plugin 107 | 1.0-RC2 108 | 109 | 110 | org.codehaus.mojo 111 | build-helper-maven-plugin 112 | 1.7 113 | 114 | 115 | org.apache.avro 116 | avro-maven-plugin 117 | ${avro.version} 118 | 119 | 120 | 121 | 122 | 123 | org.codehaus.mojo 124 | appassembler-maven-plugin 125 | 126 | 127 | 128 | Kira 129 | kira 130 | 131 | 132 | 133 | 134 | 135 | package 136 | 137 | assemble 138 | 139 | 140 | 141 | 142 | 143 | net.alchim31.maven 144 | scala-maven-plugin 145 | 146 | 147 | scala-compile-first 148 | process-resources 149 | 150 | compile 151 | 152 | 153 | 154 | -g:vars 155 | 156 | 157 | 158 | 159 | scala-test-compile-first 160 | process-test-resources 161 | 162 | testCompile 163 | 164 | 165 | 166 | attach-scaladocs 167 | verify 168 | 169 | doc-jar 170 | 171 | 172 | 173 | 174 | ${scala.version} 175 | incremental 176 | true 177 | 178 | -unchecked 179 | -optimise 180 | -deprecation 181 | 182 | 183 | -Xms64m 184 | -Xms1024m 185 | -Xmx1024m 186 | 187 | 188 | -source 189 | ${java.version} 190 | -target 191 | ${java.version} 192 | 193 | 194 | 195 | 196 | org.scalariform 197 | scalariform-maven-plugin 198 | 0.1.4 199 | 200 | 201 | default-cli 202 | 203 | format 204 | 205 | 206 | true 207 | true 208 | true 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | org.esa 220 | fits 221 | 0.94 222 | 223 | 224 | com.esotericsoftware.kryo 225 | kryo 226 | 2.21 227 | 228 | 229 | com.twitter 230 | chill-avro 231 | ${chill.version} 232 | 233 | 234 | com.twitter 235 | chill_2.11 236 | 237 | 238 | com.twitter 239 | chill-bijection_2.11 240 | 241 | 242 | com.twitter 243 | bijection-avro_2.11 244 | 245 | 246 | 247 | 248 | com.twitter 249 | bijection-avro_${scala.artifact.suffix} 250 | 0.7.0 251 | 252 | 253 | com.twitter 254 | chill-bijection_${scala.artifact.suffix} 255 | ${chill.version} 256 | 257 | 258 | com.twitter 259 | chill_${scala.artifact.suffix} 260 | ${chill.version} 261 | 262 | 263 | org.apache.hadoop 264 | hadoop-client 265 | ${hadoop.version} 266 | 267 | 268 | asm 269 | asm 270 | 271 | 272 | org.jboss.netty 273 | netty 274 | 275 | 276 | org.codehaus.jackson 277 | * 278 | 279 | 280 | org.sonatype.sisu.inject 281 | * 282 | 283 | 284 | javax.servlet 285 | servlet-api 286 | 287 | 288 | 289 | 290 | org.apache.spark 291 | spark-core_${scala.artifact.suffix} 292 | ${spark.version} 293 | 294 | 295 | org.apache.hadoop 296 | hadoop-client 297 | 298 | 299 | org.twitter 300 | chill_${scala.artifact.suffix} 301 | 302 | 303 | org.apache.hadoop 304 | hadoop-mapreduce 305 | 306 | 307 | 308 | 309 | org.apache.avro 310 | avro 311 | ${avro.version} 312 | 313 | 314 | com.twitter 315 | parquet-avro 316 | ${parquet.version} 317 | 318 | 319 | 320 | 321 | 322 | 323 | org.esa 324 | fits 325 | 326 | 327 | org.apache.spark 328 | spark-core_${scala.artifact.suffix} 329 | 330 | 331 | org.apache.hadoop 332 | hadoop-client 333 | 334 | 335 | org.apache.avro 336 | avro 337 | 338 | 339 | com.twitter 340 | parquet-avro 341 | 342 | 343 | com.twitter 344 | chill-avro 345 | 346 | 347 | com.twitter 348 | bijection-avro_${scala.artifact.suffix} 349 | 350 | 351 | com.twitter 352 | chill-bijection_${scala.artifact.suffix} 353 | 354 | 355 | com.twitter 356 | chill_${scala.artifact.suffix} 357 | 358 | 359 | 360 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/Background.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "Background.h" 6 | #include "sep.h" 7 | 8 | void sep_back_c_java(JNIEnv *env, jclass cls, jobject obj, jclass bkmapcls, jobject bkmap, sepbackmap *p){ 9 | /*set the globalback and globalrms field as return value*/ 10 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 11 | assert(id_gback != NULL); 12 | (*env)->SetFloatField(env, bkmap, id_gback, p->globalback); 13 | 14 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 15 | assert(id_rms != NULL); 16 | (*env)->SetFloatField(env, bkmap, id_rms, p->globalrms); 17 | 18 | /*set the w and h field as return value*/ 19 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 20 | (*env)->SetIntField(env, bkmap, id_w, p->w); 21 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 22 | (*env)->SetIntField(env, bkmap, id_h, p->h); 23 | 24 | /*set the bw and bh field as return value*/ 25 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 26 | (*env)->SetIntField(env, bkmap, id_bw, p->bw); 27 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 28 | (*env)->SetIntField(env, bkmap, id_bh, p->bh); 29 | 30 | /*set the nx, ny, and n field as return value*/ 31 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 32 | (*env)->SetIntField(env, bkmap, id_nx, p->nx); 33 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 34 | (*env)->SetIntField(env, bkmap, id_ny, p->ny); 35 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 36 | (*env)->SetIntField(env, bkmap, id_n, p->n); 37 | 38 | jmethodID setBackID = (*env)->GetMethodID(env, bkmapcls, "setBack", "([F)V"); 39 | jfloatArray backArray = (*env)->NewFloatArray(env, p->n); 40 | (*env)->SetFloatArrayRegion(env, backArray, 0, p->n, p->back); 41 | (*env)->CallVoidMethod(env, bkmap, setBackID, backArray); 42 | 43 | jmethodID setDbackID = (*env)->GetMethodID(env, bkmapcls, "setDback", "([F)V"); 44 | jfloatArray dbackArray = (*env)->NewFloatArray(env, p->n); 45 | (*env)->SetFloatArrayRegion(env, dbackArray, 0, p->n, p->dback); 46 | (*env)->CallVoidMethod(env, bkmap, setDbackID, dbackArray); 47 | 48 | jmethodID setSigmaID = (*env)->GetMethodID(env, bkmapcls, "setSigma", "([F)V"); 49 | jfloatArray sigmaArray = (*env)->NewFloatArray(env, p->n); 50 | (*env)->SetFloatArrayRegion(env, sigmaArray, 0, p->n, p->sigma); 51 | (*env)->CallVoidMethod(env, bkmap, setSigmaID, sigmaArray); 52 | 53 | jmethodID setDsigmaID = (*env)->GetMethodID(env, bkmapcls, "setDsigma", "([F)V"); 54 | jfloatArray dsigmaArray = (*env)->NewFloatArray(env, p->n); 55 | (*env)->SetFloatArrayRegion(env, dsigmaArray, 0, p->n, p->dsigma); 56 | (*env)->CallVoidMethod(env, bkmap, setDsigmaID, dsigmaArray); 57 | } 58 | 59 | void sep_back_java_c(JNIEnv *env, jclass bkmapcls, jobject bkmap, sepbackmap *p, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 60 | jfieldID id_h = (*env)->GetFieldID(env, bkmapcls, "h", "I"); 61 | jint h = (*env)->GetIntField(env, bkmap, id_h); 62 | jfieldID id_w = (*env)->GetFieldID(env, bkmapcls, "w", "I"); 63 | jint w = (*env)->GetIntField(env, bkmap, id_w); 64 | 65 | jfieldID id_bh = (*env)->GetFieldID(env, bkmapcls, "bh", "I"); 66 | jint bh = (*env)->GetIntField(env, bkmap, id_bh); 67 | jfieldID id_bw = (*env)->GetFieldID(env, bkmapcls, "bw", "I"); 68 | jint bw = (*env)->GetIntField(env, bkmap, id_bw); 69 | 70 | jfieldID id_nx = (*env)->GetFieldID(env, bkmapcls, "nx", "I"); 71 | jint nx = (*env)->GetIntField(env, bkmap, id_nx); 72 | jfieldID id_ny = (*env)->GetFieldID(env, bkmapcls, "ny", "I"); 73 | jint ny = (*env)->GetIntField(env, bkmap, id_ny); 74 | jfieldID id_n = (*env)->GetFieldID(env, bkmapcls, "n", "I"); 75 | jint n = (*env)->GetIntField(env, bkmap, id_n); 76 | 77 | jfieldID id_gback = (*env)->GetFieldID(env, bkmapcls, "globalback", "F"); 78 | jfloat gback = (*env)->GetFloatField(env, bkmap, id_gback); 79 | jfieldID id_rms = (*env)->GetFieldID(env, bkmapcls, "globalrms", "F"); 80 | jfloat rms = (*env)->GetFloatField(env, bkmap, id_rms); 81 | 82 | /*need to implement new array access functions here*/ 83 | jfloat *backarray = (jfloat *)(*env)->GetFloatArrayElements(env, back, NULL); 84 | jfloat *dbackarray = (jfloat *)(*env)->GetFloatArrayElements(env, dback, NULL); 85 | jfloat *sigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, sigma, NULL); 86 | jfloat *dsigmaarray = (jfloat *)(*env)->GetFloatArrayElements(env, dsigma, NULL); 87 | 88 | p->h = h; 89 | p->w = w; 90 | p->bh = bh; 91 | p->bw = bw; 92 | p->nx = nx; 93 | p->ny = ny; 94 | p->n = n; 95 | p->globalback = gback; 96 | p->globalrms = rms; 97 | 98 | jfloat *bp = p->back; 99 | jfloat *dbp = p->dback; 100 | jfloat *sp = p->sigma; 101 | jfloat *dsp = p->dsigma; 102 | 103 | memcpy(bp, backarray, sizeof(float)*(p->n)); 104 | memcpy(dbp, dbackarray, sizeof(float)*(p->n)); 105 | memcpy(sp, sigmaarray, sizeof(float)*(p->n)); 106 | memcpy(dsp, dsigmaarray, sizeof(float)*(p->n)); 107 | } 108 | 109 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 110 | (JNIEnv *env, jobject obj, jbyteArray data, jobjectArray maskdata, jint dtype, jint mdtype, jint w, jint h, jint 111 | bw, jint bh, jdouble mthresh, jint fw, jint fh, jdouble fthresh, jobject backmap){ 112 | 113 | int len = (*env)->GetArrayLength(env, data); 114 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 115 | 116 | jboolean *mask; 117 | jbooleanArray dim = NULL; 118 | jbooleanArray oneDim = NULL; 119 | if(maskdata == NULL){ 120 | mask = (void *)NULL; 121 | } 122 | else{ 123 | /*parse the mask matrix*/ 124 | int mlen1 = (*env)->GetArrayLength(env, maskdata); 125 | dim = (jbooleanArray)(*env)->GetObjectArrayElement(env, maskdata, 0); 126 | int mlen2 = (*env)->GetArrayLength(env, dim); 127 | printf("mlen2: %d\n", mlen2); 128 | mask = malloc(sizeof(jboolean)* mlen1 * mlen2); 129 | for(int i=0; iGetObjectArrayElement(env, maskdata, i); 131 | jboolean *element=(*env)->GetBooleanArrayElements(env, oneDim, 0); 132 | memcpy(mask+i*mlen2, element, sizeof(jboolean)*mlen2); 133 | } 134 | } 135 | 136 | 137 | /*printf("%d\n", len); 138 | for(int i=0; iw = w; 144 | p->h = h; 145 | p->bw = bw; 146 | p->bh = bh; 147 | 148 | int status = sep_makeback(array, mask, dtype, mdtype, w, h, bw, bh, mthresh, fw, fh, fthresh, &p); 149 | 150 | //printf("%d\n", status); 151 | printf("C sep_makeback: %d\t%d\t%f\t%f\t%d\t%d\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh, p->nx, p->ny); 152 | printf("C sep_makeback back: %f\t dback: %f\n", *(p->back), *(p->dback)); 153 | 154 | /*Finding the sepbackmap field in the Background class*/ 155 | jclass cls = (*env)->GetObjectClass(env, obj); 156 | assert(cls != NULL); 157 | jfieldID id_bkmap = (*env)->GetFieldID(env, cls, "bkgmap", "LSepbackmap;"); 158 | assert(id_bkmap != NULL); 159 | jobject bkmap = (*env)->GetObjectField(env, obj, id_bkmap); 160 | assert(bkmap != NULL); 161 | jclass bkmapcls = (*env)->GetObjectClass(env, bkmap); 162 | assert(bkmapcls != NULL); 163 | 164 | sep_back_c_java(env, cls, obj, bkmapcls, bkmap, p); 165 | 166 | float *fp = p->back; 167 | for(int i=0; in; i++){ 168 | printf("back: %f\t", *(fp++)); 169 | } 170 | printf("\n"); 171 | 172 | fp = p->dback; 173 | for(int i=0; in; i++){ 174 | printf("dback: %f\t", *(fp++)); 175 | } 176 | printf("\n"); 177 | 178 | fp = p->sigma; 179 | for(int i=0; in; i++){ 180 | printf("sigma: %f\t", *(fp++)); 181 | } 182 | printf("\n"); 183 | 184 | fp = p->dsigma; 185 | for(int i=0; in; i++){ 186 | printf("dsigma: %f\t", *(fp++)); 187 | } 188 | printf("\n"); 189 | free(array); 190 | free(p); 191 | 192 | if(mask != NULL){ 193 | printf("C makeback: free mask\n"); 194 | free(mask); 195 | } 196 | 197 | return status; 198 | } 199 | 200 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 201 | (JNIEnv *env, jobject obj, jobject bkmap, jbyteArray data, jint dtype, jfloatArray back, jfloatArray dback, jfloatArray sigma, jfloatArray dsigma){ 202 | 203 | int len = (*env)->GetArrayLength(env, data); 204 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 205 | 206 | jclass cls = (*env)->GetObjectClass(env, bkmap); 207 | assert(cls != NULL); 208 | 209 | int backLen = (*env)->GetArrayLength(env, back); 210 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 211 | p->back = malloc(sizeof(float)*backLen); 212 | p->dback = malloc(sizeof(float)*backLen); 213 | p->sigma = malloc(sizeof(float)*backLen); 214 | p->dsigma = malloc(sizeof(float)*backLen); 215 | 216 | sep_back_java_c(env, cls, bkmap, p, back, dback, sigma, dsigma); 217 | 218 | printf("C sep_backarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 219 | printf("C sep_backarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 220 | printf("C sep_backarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 221 | printf("C sep_backarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma)); 222 | 223 | 224 | int status = sep_backarray(p, array, dtype); 225 | printf("status: %d\n", status); 226 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 227 | 228 | printf("%d, \n", array[0]); 229 | 230 | /*for(int i=0; iGetArrayLength(env, data); 246 | jbyte *array = (jbyte *)(*env)->GetByteArrayElements(env, data, NULL); 247 | 248 | jclass cls = (*env)->GetObjectClass(env, bkgmap); 249 | assert(cls != NULL); 250 | 251 | int backLen = (*env)->GetArrayLength(env, back); 252 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 253 | p->back = malloc(sizeof(float)*backLen); 254 | p->dback = malloc(sizeof(float)*backLen); 255 | p->sigma = malloc(sizeof(float)*backLen); 256 | p->dsigma = malloc(sizeof(float)*backLen); 257 | 258 | sep_back_java_c(env, cls, bkgmap, p, back, dback, sigma, dsigma); 259 | 260 | printf("C sep_subbackarray: %d\t%d\t%f\t%f\t%d\t%d\n", p->w, p->h, p->globalback, p->globalrms, p->bw, p->bh); 261 | printf("C sep_subbackarray: nx: %d \t ny: %d \t n: %d\n", p->nx, p->ny, p->n); 262 | printf("C sep_subbackarray: back: %f \t dback: %f\n", *(p->back), *(p->dback)); 263 | printf("C sep_subbackarray: sigma: %f \t dsigma: %f\n", *(p->sigma), *(p->dsigma)); 264 | 265 | int status = sep_subbackarray(p, array, dtype); 266 | (*env)->SetByteArrayRegion(env, data, 0, len, array); 267 | 268 | /*printf("C sep_subbackarray: result\n"); 269 | double *dp = (double *)array; 270 | for(int i=0; i 3 | /* Header for class Background */ 4 | 5 | #ifndef _Included_Background 6 | #define _Included_Background 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Background 12 | * Method: sep_makeback 13 | * Signature: ([B[[ZIIIIIIDIIDLBackground/Sepbackmap;)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 16 | (JNIEnv *, jobject, jbyteArray, jobjectArray, jint, jint, jint, jint, jint, jint, jdouble, jint, jint, jdouble, jobject); 17 | 18 | /* 19 | * Class: Background 20 | * Method: sep_backarray 21 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 24 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 25 | 26 | /* 27 | * Class: Background 28 | * Method: sep_backrmsarray 29 | * Signature: (LBackground/Sepbackmap;[Ljava/lang/Object;I)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Background_sep_1backrmsarray 32 | (JNIEnv *, jobject, jobject, jobjectArray, jint); 33 | 34 | /* 35 | * Class: Background 36 | * Method: sep_subbackarray 37 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Background_sep_1subbackarray 40 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 41 | 42 | /* 43 | * Class: Background 44 | * Method: sep_freeback 45 | * Signature: (LBackground/Sepbackmap;)V 46 | */ 47 | JNIEXPORT void JNICALL Java_Background_sep_1freeback 48 | (JNIEnv *, jobject, jobject); 49 | 50 | /* 51 | * Class: Background 52 | * Method: sep_extract 53 | * Signature: ([B[BIISIIFI[BIIIDZD[LBackground/Sepobj;I)I 54 | */ 55 | JNIEXPORT jint JNICALL Java_Background_sep_1extract 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 57 | 58 | /* 59 | * Class: Background 60 | * Method: sep_freeobjarray 61 | * Signature: ([LBackground/Sepobj;I)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Background_sep_1freeobjarray 64 | (JNIEnv *, jobject, jobjectArray, jint); 65 | 66 | /* 67 | * Class: Background 68 | * Method: sep_sum_circle 69 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 70 | */ 71 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circle 72 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 73 | 74 | /* 75 | * Class: Background 76 | * Method: sep_sum_circann 77 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 78 | */ 79 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circann 80 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 81 | 82 | /* 83 | * Class: Background 84 | * Method: sep_sum_ellipse 85 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 86 | */ 87 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipse 88 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 89 | 90 | /* 91 | * Class: Background 92 | * Method: sep_sum_ellipann 93 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 94 | */ 95 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipann 96 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 97 | 98 | /* 99 | * Class: Background 100 | * Method: sep_kron_radius 101 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 102 | */ 103 | JNIEXPORT void JNICALL Java_Background_sep_1kron_1radius 104 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 105 | 106 | /* 107 | * Class: Background 108 | * Method: sep_ellipse_axes 109 | * Signature: ([D[D[D[D[D[D)V 110 | */ 111 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1axes 112 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 113 | 114 | /* 115 | * Class: Background 116 | * Method: sep_ellipse_coeffs 117 | * Signature: ([D[D[D[D[D[D)V 118 | */ 119 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1coeffs 120 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 121 | 122 | /* 123 | * Class: Background 124 | * Method: sep_set_ellipse 125 | * Signature: (Ljava/lang/String;IIDDDDDDS)V 126 | */ 127 | JNIEXPORT void JNICALL Java_Background_sep_1set_1ellipse 128 | (JNIEnv *, jobject, jstring, jint, jint, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jshort); 129 | 130 | /* 131 | * Class: Background 132 | * Method: sep_set_extract_pixstack 133 | * Signature: (I)V 134 | */ 135 | JNIEXPORT void JNICALL Java_Background_sep_1set_1extract_1pixstack 136 | (JNIEnv *, jobject, jint); 137 | 138 | /* 139 | * Class: Background 140 | * Method: sep_get_extract_pixstack 141 | * Signature: ()I 142 | */ 143 | JNIEXPORT jint JNICALL Java_Background_sep_1get_1extract_1pixstack 144 | (JNIEnv *, jobject); 145 | 146 | /* 147 | * Class: Background 148 | * Method: sep_get_errmsg 149 | * Signature: (ILjava/lang/String;)V 150 | */ 151 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errmsg 152 | (JNIEnv *, jobject, jint, jstring); 153 | 154 | /* 155 | * Class: Background 156 | * Method: sep_get_errdetail 157 | * Signature: (Ljava/lang/String;)V 158 | */ 159 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errdetail 160 | (JNIEnv *, jobject, jstring); 161 | 162 | #ifdef __cplusplus 163 | } 164 | #endif 165 | #endif 166 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/Background_Sepbackmap.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepbackmap */ 4 | 5 | #ifndef _Included_Background_Sepbackmap 6 | #define _Included_Background_Sepbackmap 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/Background_Sepobj.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepobj */ 4 | 5 | #ifndef _Included_Background_Sepobj 6 | #define _Included_Background_Sepobj 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/Extractor.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Extractor */ 4 | 5 | #ifndef _Included_Extractor 6 | #define _Included_Extractor 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Extractor 12 | * Method: sep_sum_circle 13 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circle 16 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 17 | 18 | /* 19 | * Class: Extractor 20 | * Method: sep_sum_circann 21 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1circann 24 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 25 | 26 | /* 27 | * Class: Extractor 28 | * Method: sep_sum_ellipse 29 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipse 32 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 33 | 34 | /* 35 | * Class: Extractor 36 | * Method: sep_sum_ellipann 37 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Extractor_sep_1sum_1ellipann 40 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 41 | 42 | /* 43 | * Class: Extractor 44 | * Method: sep_extract 45 | * Signature: ([B[BIISIIFI[BIIIDZD[LSepobj;I)I 46 | */ 47 | JNIEXPORT jint JNICALL Java_Extractor_sep_1extract 48 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 49 | 50 | /* 51 | * Class: Extractor 52 | * Method: sep_kron_radius 53 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 54 | */ 55 | JNIEXPORT void JNICALL Java_Extractor_sep_1kron_1radius 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 57 | 58 | /* 59 | * Class: Extractor 60 | * Method: sep_ellipse_coeffs 61 | * Signature: ([D[D[D[D[D[D)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1coeffs 64 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 65 | 66 | /* 67 | * Class: Extractor 68 | * Method: sep_ellipse_axes 69 | * Signature: ([D[D[D[D[D[D)V 70 | */ 71 | JNIEXPORT void JNICALL Java_Extractor_sep_1ellipse_1axes 72 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 73 | 74 | /* 75 | * Class: Extractor 76 | * Method: sep_set_ellipse 77 | * Signature: ([BII[D[D[D[D[D[DB)V 78 | */ 79 | JNIEXPORT void JNICALL Java_Extractor_sep_1set_1ellipse 80 | (JNIEnv *, jobject, jbyteArray, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jbyte); 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | #endif 86 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/README.md: -------------------------------------------------------------------------------- 1 | gcc Background.c Extractor.c -o libBackgroundImpl.jnilib -lc -shared -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include" -I"/Library/Java/JavaVirtualMachines/jdk1.7.0_60.jdk/Contents/Home/include/darwin" -L"/Users/zhaozhang/projects/scratch/java/test/libs" -lsep -I"/Users/zhaozhang/projects/scratch/java/test/libs/include" 2 | -------------------------------------------------------------------------------- /scratch/spark/src/main/C/test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "sep.h" 4 | 5 | int main(){ 6 | double darray[6][6]; 7 | for(int i=0; i<6; i++) 8 | for(int j=0; j<6; j++) 9 | darray[i][j] = 0.0; 10 | 11 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 12 | p->back = malloc(sizeof(float)*4); 13 | p->dback = malloc(sizeof(float)*4); 14 | p->sigma = malloc(sizeof(float)*4); 15 | p->dsigma = malloc(sizeof(float)*4); 16 | 17 | p->h = 6; 18 | p->w = 6; 19 | p->bh = 3; 20 | p->bw = 3; 21 | p->nx = 2; 22 | p->ny = 2; 23 | p->n = 4; 24 | p->globalback = 0.08958334; 25 | p->globalrms = 0.3142697; 26 | 27 | float *pp = p->back; 28 | for(int i=0; in; i++){ 29 | *(pp++) = 0.08958334; 30 | } 31 | 32 | pp = p->dback; 33 | for(int i=0; in; i++){ 34 | *(pp++) = 0.0; 35 | } 36 | 37 | pp = p->sigma; 38 | for(int i=0; in; i++){ 39 | *(pp++) = 0.3142697; 40 | } 41 | 42 | pp = p->dsigma; 43 | for(int i=0; in; i++){ 44 | *(pp++) = 0.0; 45 | } 46 | 47 | int status = sep_backarray(p, darray, 82); 48 | printf("%f, \n", darray[0][0]); 49 | } -------------------------------------------------------------------------------- /scratch/spark/src/main/scala/Kira.scala: -------------------------------------------------------------------------------- 1 | import org.apache.spark.SparkContext 2 | import org.apache.spark.SparkContext._ 3 | import org.apache.spark.SparkConf 4 | import java.io._ 5 | import java.util.zip.CRC32 6 | import java.util.zip.Checksum 7 | 8 | object Kira { 9 | def main(args: Array[String]) { 10 | val conf = new SparkConf().setAppName("Kira") 11 | val sc = new SparkContext(conf) 12 | 13 | val src = "/Users/zhaozhang/projects/scratch/Kira/data/" 14 | val flist = sc.binaryFiles(src) 15 | 16 | val results = flist.map(c => extract_str(c._2.toArray)) 17 | val flatresults = results.flatMap(p => p.map(r => (r._1, r._2, r._3, r._4, r._5))) 18 | println("count: " + flatresults.count) 19 | flatresults.saveAsTextFile("output") 20 | 21 | /*val flist = new File(src).listFiles.filter(_.getName.endsWith(".fits")) 22 | 23 | val dflist = sc.parallelize(flist) 24 | val results = dflist.map(f => extract(f.toString)) 25 | 26 | val flatresults = results.flatMap(p => p.map(r => (r._1, r._2, r._3, r._4, r._5))) 27 | println("count: " + flatresults.count) 28 | flatresults.saveAsTextFile("output")*/ 29 | } 30 | def extract(path: String): Array[(Double, Double, Double, Double, Short)] = { 31 | var matrix = Utils.load(path) 32 | var bkg = new Background(matrix) 33 | matrix = bkg.subfrom(matrix) 34 | val ex = new Extractor 35 | val objects = ex.extract(matrix, (1.5 * bkg.bkgmap.globalrms).toFloat) 36 | //return objects.length 37 | var x: Array[Double] = Array.ofDim[Double](objects.length) 38 | var y: Array[Double] = Array.ofDim[Double](objects.length) 39 | for (i <- (0 until objects.length)) { 40 | x(i) = objects(i).x 41 | y(i) = objects(i).y 42 | } 43 | 44 | var err: Array[Array[Double]] = Array.fill(matrix.length, matrix(0).length) { bkg.bkgmap.globalrms } 45 | 46 | val (flux, fluxerr, flag) = ex.sum_circle(matrix, x, y, 5.0, err = err) 47 | val retArray = (0 until objects.length).map(i => (x(i), y(i), flux(i), fluxerr(i), flag(i))).toArray 48 | return retArray 49 | } 50 | 51 | def extract_str(content: Array[Byte]): Array[(Double, Double, Double, Double, Short)] = { 52 | var matrix = Utils.load_byte(content) 53 | var bkg = new Background(matrix) 54 | matrix = bkg.subfrom(matrix) 55 | val ex = new Extractor 56 | val objects = ex.extract(matrix, (1.5 * bkg.bkgmap.globalrms).toFloat) 57 | //return objects.length 58 | var x: Array[Double] = Array.ofDim[Double](objects.length) 59 | var y: Array[Double] = Array.ofDim[Double](objects.length) 60 | for (i <- (0 until objects.length)) { 61 | x(i) = objects(i).x 62 | y(i) = objects(i).y 63 | } 64 | 65 | var err: Array[Array[Double]] = Array.fill(matrix.length, matrix(0).length) { bkg.bkgmap.globalrms } 66 | 67 | val (flux, fluxerr, flag) = ex.sum_circle(matrix, x, y, 5.0, err = err) 68 | val retArray = (0 until objects.length).map(i => (x(i), y(i), flux(i), fluxerr(i), flag(i))).toArray 69 | return retArray 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /scratch/spark/src/main/scala/Sepbackmap.scala: -------------------------------------------------------------------------------- 1 | class Sepbackmap(var w: Int, var h: Int, var globalback: Float, var globalrms: Float, var bw: Int, 2 | var bh: Int, var nx: Int, var ny: Int, var n: Int, var back: Array[Float], var dback: Array[Float], 3 | var sigma: Array[Float], var dsigma: Array[Float]) { 4 | back = new Array[Float](n) 5 | dback = new Array[Float](n) 6 | sigma = new Array[Float](n) 7 | dsigma = new Array[Float](n) 8 | 9 | def this() { 10 | this(0, 0, 0.0F, 0.0F, 0, 0, 0, 0, 0, null, null, null, null) 11 | } 12 | 13 | def setBack(vback: Array[Float]) { 14 | back = new Array[Float](vback.length) 15 | for (i <- (0 until back.length)) { 16 | back(i) = vback(i) 17 | } 18 | } 19 | 20 | def setDback(vdback: Array[Float]) { 21 | dback = new Array[Float](vdback.length) 22 | for (i <- (0 until dback.length)) { 23 | dback(i) = vdback(i) 24 | } 25 | } 26 | 27 | def setSigma(vsigma: Array[Float]) { 28 | sigma = new Array[Float](vsigma.length) 29 | for (i <- (0 until sigma.length)) { 30 | sigma(i) = vsigma(i) 31 | } 32 | } 33 | 34 | def setDsigma(vdsigma: Array[Float]) { 35 | dsigma = new Array[Float](vdsigma.length) 36 | for (i <- (0 until dsigma.length)) { 37 | dsigma(i) = vdsigma(i) 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /scratch/spark/src/main/scala/Sepobj.scala: -------------------------------------------------------------------------------- 1 | 2 | class Sepobj(var thresh: Double, var npix: Int, var tnpix: Int, var xmin: Int, var xmax: Int, 3 | var ymin: Int, var ymax: Int, var x: Double, var y: Double, var x2: Double, var y2: Double, 4 | var xy: Double, var a: Float, var b: Float, var theta: Float, var cxx: Float, var cyy: Float, 5 | var cxy: Float, var cflux: Float, var flux: Float, var cpeak: Float, var peak: Float, 6 | var xpeak: Float, var ypeak: Float, var xcpeak: Float, var ycpeak: Float, var flag: Short, var pix: Int) -------------------------------------------------------------------------------- /scratch/spark/src/main/scala/Utils.scala: -------------------------------------------------------------------------------- 1 | import java.lang.{ Double => jDouble } 2 | import java.nio.ByteBuffer 3 | import java.io._ 4 | import org.eso.fits._ 5 | 6 | object Utils { 7 | /**Helper function that converts 2d double matrix to 1d byte stream*/ 8 | def flatten(matrix: Array[Array[Double]]): Array[Byte] = { 9 | val h = matrix.length 10 | val w = matrix(0).length 11 | var stream = new Array[Byte](w * h * 8) 12 | 13 | for (i <- (0 until h)) { 14 | for (j <- (0 until w)) { 15 | var l: Long = jDouble.doubleToLongBits(matrix(i)(j)) 16 | for (k <- (0 until 8)) { 17 | stream((i * w + j) * 8 + k) = ((l >>> (k * 8)) & 0xff).toByte 18 | } 19 | } 20 | } 21 | stream 22 | } 23 | 24 | /**Helper function that converts 1d byte stream to 2d double matrix*/ 25 | def deflatten(data: Array[Byte], h: Int, w: Int): Array[Array[Double]] = { 26 | var matrix = Array.ofDim[Double](h, w) 27 | for (i <- (0 until h)) { 28 | for (j <- (0 until w)) { 29 | var bytes = new Array[Byte](8) 30 | for (k <- (0 until 8)) { 31 | bytes(7 - k) = (data((i * w + j) * 8 + k) & 0xff).toByte 32 | } 33 | matrix(i)(j) = ByteBuffer.wrap(bytes).getDouble() 34 | } 35 | } 36 | return matrix 37 | } 38 | 39 | def load(path: String): Array[Array[Double]] = { 40 | val file: FitsFile = new FitsFile(path) 41 | val hdu: FitsHDUnit = file.getHDUnit(0) 42 | val dm: FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 43 | val naxis: Array[Int] = dm.getNaxis() 44 | val ncol = naxis(0) 45 | val nval = dm.getNoValues() 46 | val nrow = nval / ncol 47 | 48 | // build and populate an array 49 | var matrix = Array.ofDim[Float](nrow, ncol) 50 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 51 | 52 | println("matrix: height: " + nrow + "\twidth: " + ncol) 53 | 54 | var rmatrix = Array.ofDim[Double](nrow, ncol) 55 | for (i <- (0 until nrow)) { 56 | for (j <- (0 until ncol)) { 57 | rmatrix(i)(j) = matrix(i)(j).toDouble 58 | } 59 | } 60 | 61 | return rmatrix 62 | } 63 | 64 | def load_byte(content: Array[Byte]): Array[Array[Double]] = { 65 | val is = new ByteArrayInputStream(content) 66 | val dis = new DataInputStream(is) 67 | val file = new FitsFile(dis, true) 68 | val hdu: FitsHDUnit = file.getHDUnit(0) 69 | val dm: FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 70 | val naxis: Array[Int] = dm.getNaxis() 71 | val ncol = naxis(0) 72 | val nval = dm.getNoValues() 73 | val nrow = nval / ncol 74 | println("nrow: " + nrow + "\t ncol:" + ncol) 75 | // build and populate an array 76 | var matrix = Array.ofDim[Float](nrow, ncol) 77 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 78 | 79 | println("matrix: height: " + nrow + "\twidth: " + ncol) 80 | 81 | var rmatrix = Array.ofDim[Double](nrow, ncol) 82 | for (i <- (0 until nrow)) { 83 | for (j <- (0 until ncol)) { 84 | rmatrix(i)(j) = matrix(i)(j).toDouble 85 | } 86 | } 87 | 88 | return rmatrix 89 | } 90 | } -------------------------------------------------------------------------------- /scratch/spark/src/main/scala/libBackgroundImpl.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BIDS/Kira/870c2364088aef8b114c87aab11c42ff01184cad/scratch/spark/src/main/scala/libBackgroundImpl.jnilib -------------------------------------------------------------------------------- /scratch/test/Background.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background */ 4 | 5 | #ifndef _Included_Background 6 | #define _Included_Background 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: Background 12 | * Method: sep_makeback 13 | * Signature: ([B[[ZIIIIIIDIIDLBackground/Sepbackmap;)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_Background_sep_1makeback 16 | (JNIEnv *, jobject, jbyteArray, jobjectArray, jint, jint, jint, jint, jint, jint, jdouble, jint, jint, jdouble, jobject); 17 | 18 | /* 19 | * Class: Background 20 | * Method: sep_backarray 21 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 22 | */ 23 | JNIEXPORT jint JNICALL Java_Background_sep_1backarray 24 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 25 | 26 | /* 27 | * Class: Background 28 | * Method: sep_backrmsarray 29 | * Signature: (LBackground/Sepbackmap;[Ljava/lang/Object;I)I 30 | */ 31 | JNIEXPORT jint JNICALL Java_Background_sep_1backrmsarray 32 | (JNIEnv *, jobject, jobject, jobjectArray, jint); 33 | 34 | /* 35 | * Class: Background 36 | * Method: sep_subbackarray 37 | * Signature: (LBackground/Sepbackmap;[BI[F[F[F[F)I 38 | */ 39 | JNIEXPORT jint JNICALL Java_Background_sep_1subbackarray 40 | (JNIEnv *, jobject, jobject, jbyteArray, jint, jfloatArray, jfloatArray, jfloatArray, jfloatArray); 41 | 42 | /* 43 | * Class: Background 44 | * Method: sep_freeback 45 | * Signature: (LBackground/Sepbackmap;)V 46 | */ 47 | JNIEXPORT void JNICALL Java_Background_sep_1freeback 48 | (JNIEnv *, jobject, jobject); 49 | 50 | /* 51 | * Class: Background 52 | * Method: sep_extract 53 | * Signature: ([B[BIISIIFI[BIIIDZD[LBackground/Sepobj;I)I 54 | */ 55 | JNIEXPORT jint JNICALL Java_Background_sep_1extract 56 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jshort, jint, jint, jfloat, jint, jbyteArray, jint, jint, jint, jdouble, jboolean, jdouble, jobjectArray, jint); 57 | 58 | /* 59 | * Class: Background 60 | * Method: sep_freeobjarray 61 | * Signature: ([LBackground/Sepobj;I)V 62 | */ 63 | JNIEXPORT void JNICALL Java_Background_sep_1freeobjarray 64 | (JNIEnv *, jobject, jobjectArray, jint); 65 | 66 | /* 67 | * Class: Background 68 | * Method: sep_sum_circle 69 | * Signature: ([B[B[BIIIIIDDS[D[DDI[D[D[D[S)I 70 | */ 71 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circle 72 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 73 | 74 | /* 75 | * Class: Background 76 | * Method: sep_sum_circann 77 | * Signature: ([B[B[BIIIIIDDS[D[DDDI[D[D[D[S)I 78 | */ 79 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1circann 80 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 81 | 82 | /* 83 | * Class: Background 84 | * Method: sep_sum_ellipse 85 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDI[D[D[D[S)I 86 | */ 87 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipse 88 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 89 | 90 | /* 91 | * Class: Background 92 | * Method: sep_sum_ellipann 93 | * Signature: ([B[B[BIIIIIDDS[D[D[D[D[DDDI[D[D[D[S)I 94 | */ 95 | JNIEXPORT jint JNICALL Java_Background_sep_1sum_1ellipann 96 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jint, jdouble, jdouble, jshort, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdouble, jdouble, jint, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 97 | 98 | /* 99 | * Class: Background 100 | * Method: sep_kron_radius 101 | * Signature: ([B[BIIIID[D[D[D[D[D[D[D[S)V 102 | */ 103 | JNIEXPORT void JNICALL Java_Background_sep_1kron_1radius 104 | (JNIEnv *, jobject, jbyteArray, jbyteArray, jint, jint, jint, jint, jdouble, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jshortArray); 105 | 106 | /* 107 | * Class: Background 108 | * Method: sep_ellipse_axes 109 | * Signature: ([D[D[D[D[D[D)V 110 | */ 111 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1axes 112 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 113 | 114 | /* 115 | * Class: Background 116 | * Method: sep_ellipse_coeffs 117 | * Signature: ([D[D[D[D[D[D)V 118 | */ 119 | JNIEXPORT void JNICALL Java_Background_sep_1ellipse_1coeffs 120 | (JNIEnv *, jobject, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray, jdoubleArray); 121 | 122 | /* 123 | * Class: Background 124 | * Method: sep_set_ellipse 125 | * Signature: (Ljava/lang/String;IIDDDDDDS)V 126 | */ 127 | JNIEXPORT void JNICALL Java_Background_sep_1set_1ellipse 128 | (JNIEnv *, jobject, jstring, jint, jint, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jshort); 129 | 130 | /* 131 | * Class: Background 132 | * Method: sep_set_extract_pixstack 133 | * Signature: (I)V 134 | */ 135 | JNIEXPORT void JNICALL Java_Background_sep_1set_1extract_1pixstack 136 | (JNIEnv *, jobject, jint); 137 | 138 | /* 139 | * Class: Background 140 | * Method: sep_get_extract_pixstack 141 | * Signature: ()I 142 | */ 143 | JNIEXPORT jint JNICALL Java_Background_sep_1get_1extract_1pixstack 144 | (JNIEnv *, jobject); 145 | 146 | /* 147 | * Class: Background 148 | * Method: sep_get_errmsg 149 | * Signature: (ILjava/lang/String;)V 150 | */ 151 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errmsg 152 | (JNIEnv *, jobject, jint, jstring); 153 | 154 | /* 155 | * Class: Background 156 | * Method: sep_get_errdetail 157 | * Signature: (Ljava/lang/String;)V 158 | */ 159 | JNIEXPORT void JNICALL Java_Background_sep_1get_1errdetail 160 | (JNIEnv *, jobject, jstring); 161 | 162 | #ifdef __cplusplus 163 | } 164 | #endif 165 | #endif 166 | -------------------------------------------------------------------------------- /scratch/test/Background_Sepbackmap.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepbackmap */ 4 | 5 | #ifndef _Included_Background_Sepbackmap 6 | #define _Included_Background_Sepbackmap 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/test/Background_Sepobj.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class Background_Sepobj */ 4 | 5 | #ifndef _Included_Background_Sepobj 6 | #define _Included_Background_Sepobj 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #ifdef __cplusplus 11 | } 12 | #endif 13 | #endif 14 | -------------------------------------------------------------------------------- /scratch/test/Fits.java: -------------------------------------------------------------------------------- 1 | import org.eso.fits.*; 2 | import java.util.*; 3 | import java.io.*; 4 | 5 | public class Fits{ 6 | public static double[][] load(String path){ 7 | double[][] dmatrix = null; 8 | try{ 9 | //FitsFile file = new FitsFile("/Users/zhaozhang/projects/scratch/java/test/data/image.fits"); 10 | FitsFile file = new FitsFile(path); 11 | FitsHDUnit hdu = file.getHDUnit(0); 12 | FitsMatrix dm = (FitsMatrix)hdu.getData(); 13 | int[] naxis = dm.getNaxis(); 14 | int ncol = naxis[0]; 15 | int nval = dm.getNoValues(); 16 | int nrow = nval/ncol; 17 | 18 | float[][] matrix = new float[nrow][ncol]; 19 | for(int i=0; i"); 29 | } catch (IOException e) { 30 | System.out.println("Error: cannot open file >"); 31 | } 32 | return dmatrix; 33 | } 34 | public static void main(String[] args){ 35 | double[][] matrix = load("/Users/zhaozhang/projects/scratch/java/test/data/image.fits"); 36 | for(int i=0; i 2 | #include 3 | #include "sep.h" 4 | 5 | int main(){ 6 | double darray[6][6]; 7 | for(int i=0; i<6; i++) 8 | for(int j=0; j<6; j++) 9 | darray[i][j] = 0.0; 10 | 11 | sepbackmap *p = (sepbackmap *)malloc(sizeof(sepbackmap)); 12 | p->back = malloc(sizeof(float)*4); 13 | p->dback = malloc(sizeof(float)*4); 14 | p->sigma = malloc(sizeof(float)*4); 15 | p->dsigma = malloc(sizeof(float)*4); 16 | 17 | p->h = 6; 18 | p->w = 6; 19 | p->bh = 3; 20 | p->bw = 3; 21 | p->nx = 2; 22 | p->ny = 2; 23 | p->n = 4; 24 | p->globalback = 0.08958334; 25 | p->globalrms = 0.3142697; 26 | 27 | float *pp = p->back; 28 | for(int i=0; in; i++){ 29 | *(pp++) = 0.08958334; 30 | } 31 | 32 | pp = p->dback; 33 | for(int i=0; in; i++){ 34 | *(pp++) = 0.0; 35 | } 36 | 37 | pp = p->sigma; 38 | for(int i=0; in; i++){ 39 | *(pp++) = 0.3142697; 40 | } 41 | 42 | pp = p->dsigma; 43 | for(int i=0; in; i++){ 44 | *(pp++) = 0.0; 45 | } 46 | 47 | int status = sep_backarray(p, darray, 82); 48 | printf("%f, \n", darray[0][0]); 49 | } -------------------------------------------------------------------------------- /src/main/resources/avro/fits.avdl: -------------------------------------------------------------------------------- 1 | @namespace("edu.berkeley.bids.kira.avro") 2 | protocol Fits { 3 | 4 | record FitsValue { 5 | union { null, int } xPos = null; 6 | union { null, int } yPos = null; 7 | union { null, float } value = null; 8 | union { null, int } start = null; 9 | union { null, int } end = null; 10 | union { null, int } offset = null; 11 | union { null, int } height = null; 12 | union { null, string} path = null; 13 | } 14 | 15 | } -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/Fits2Kira.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira 17 | 18 | import edu.berkeley.bids.kira.avro.FitsValue 19 | import edu.berkeley.bids.kira.models._ 20 | import edu.berkeley.bids.kira.util.FitsUtils 21 | import java.io._ 22 | import org.apache.hadoop.conf.Configuration 23 | import org.apache.hadoop.mapreduce.Job 24 | import org.apache.spark.SparkContext._ 25 | import org.apache.spark.{ SparkConf, SparkContext } 26 | import org.apache.spark.rdd.RDD 27 | import parquet.avro.{ 28 | AvroParquetInputFormat, 29 | AvroParquetOutputFormat, 30 | AvroReadSupport 31 | } 32 | import parquet.filter.UnboundRecordFilter 33 | import parquet.hadoop.{ 34 | ParquetInputFormat, 35 | ParquetOutputFormat 36 | } 37 | import parquet.hadoop.util.ContextUtil 38 | import parquet.hadoop.metadata.CompressionCodecName 39 | import scala.annotation.tailrec 40 | 41 | object Fits2Kira extends Serializable { 42 | val usage = """ 43 | Usage: Fits2Kira [srcpath] [dstpath] 44 | """ 45 | def main(args: Array[String]) { 46 | 47 | if (args.length < 2) { 48 | print(usage) 49 | System.exit(1) 50 | } 51 | val src = args(0).toString 52 | val dst = args(1).toString 53 | 54 | def parallelize(f: Fits, 55 | metadata: FitsMetadata, 56 | sc: SparkContext): RDD[FitsValue] = { 57 | sc.parallelize(f.matrix.zipWithIndex.flatMap(vk => { 58 | val (array, idx) = vk 59 | 60 | array.zipWithIndex.map(vk2 => { 61 | val (value, jdx) = vk2 62 | 63 | FitsValue.newBuilder() 64 | .setXPos(idx) 65 | .setYPos(jdx) 66 | .setValue(value) 67 | .setStart(metadata.start) 68 | .setEnd(metadata.end) 69 | .setOffset(metadata.offset) 70 | .setPath(f.path) 71 | .build() 72 | }) 73 | })) 74 | } 75 | 76 | /** 77 | * Create a job using either the Hadoop 1 or 2 API 78 | * @param sc A Spark context 79 | */ 80 | def newJob(sc: SparkContext): Job = { 81 | newJobFromConfig(sc.hadoopConfiguration) 82 | } 83 | 84 | def newJobFromConfig(config: Configuration): Job = { 85 | val jobClass: Class[_] = Class.forName("org.apache.hadoop.mapreduce.Job") 86 | try { 87 | // Use the getInstance method in Hadoop 2 88 | jobClass.getMethod("getInstance", classOf[Configuration]).invoke(null, config).asInstanceOf[Job] 89 | } catch { 90 | case ex: NoSuchMethodException => 91 | // Drop back to Hadoop 1 constructor 92 | jobClass.getConstructor(classOf[Configuration]).newInstance(config).asInstanceOf[Job] 93 | } 94 | } 95 | 96 | def save(rdd: RDD[FitsValue], 97 | job: Job, 98 | filePath: String, 99 | blockSize: Int = 128 * 1024 * 1024, 100 | pageSize: Int = 1 * 1024 * 1024, 101 | compressCodec: CompressionCodecName = CompressionCodecName.GZIP, 102 | disableDictionaryEncoding: Boolean = false) { 103 | 104 | // configure parquet 105 | ParquetOutputFormat.setCompression(job, compressCodec) 106 | ParquetOutputFormat.setEnableDictionary(job, !disableDictionaryEncoding) 107 | ParquetOutputFormat.setBlockSize(job, blockSize) 108 | ParquetOutputFormat.setPageSize(job, pageSize) 109 | AvroParquetOutputFormat.setSchema(job, new FitsValue().getSchema) 110 | 111 | // Add the Void Key 112 | val recordToSave = rdd.map(p => (null, p)) 113 | 114 | // Save the values to the parquet file 115 | recordToSave.saveAsNewAPIHadoopFile(filePath, 116 | classOf[java.lang.Void], 117 | classOf[FitsValue], 118 | classOf[AvroParquetOutputFormat], 119 | ContextUtil.getConfiguration(job)) 120 | } 121 | 122 | val conf = new SparkConf().setAppName("Fits2Kira") 123 | if (conf.getOption("spark.master").isEmpty) { 124 | conf.setMaster("local[%d]".format(Runtime.getRuntime.availableProcessors())) 125 | } 126 | val sc = new SparkContext(conf) 127 | 128 | @tailrec def buildUp(datasets: Iterator[(Fits, FitsMetadata)], 129 | lastRdd: RDD[FitsValue] = sc.parallelize(Array[FitsValue]())): RDD[FitsValue] = { 130 | if (!datasets.hasNext) { 131 | lastRdd 132 | } else { 133 | // cache last RDD 134 | lastRdd.cache() 135 | 136 | // parallelize the current dataset 137 | val (data, metadata) = datasets.next 138 | val newRdd = parallelize(data, metadata, sc) ++ lastRdd 139 | 140 | // unpersist the old rdd and recurse 141 | lastRdd.unpersist() 142 | buildUp(datasets, newRdd) 143 | } 144 | } 145 | 146 | def add(rdd: RDD[FitsValue], 147 | tcol: Int, 148 | trow: Int): RDD[FitsValue] = { 149 | // populate matrix 150 | rdd.flatMap(v => { 151 | val coords = Coordinate(v.getXPos + v.getStart - 1, v.getYPos + v.getOffset) 152 | 153 | if (coords.x < trow && coords.y < tcol) { 154 | Some((coords, v.getValue.toFloat)) 155 | } else { 156 | None 157 | } 158 | }).groupByKey() 159 | .map(kv => { 160 | val (idx, values) = kv 161 | val l = values.filter(x => !x.isNaN) 162 | val v = if (l.size > 0) l.reduce(_ + _) / l.size else Float.NaN 163 | 164 | FitsValue.newBuilder() 165 | .setXPos(idx.x) 166 | .setYPos(idx.y) 167 | .setValue(v) 168 | .setStart(0) 169 | .setEnd(tcol) 170 | .setHeight(trow) 171 | .setOffset(0) 172 | .build() 173 | }) 174 | } 175 | 176 | def buildMatrix(rdd: RDD[FitsValue]): Array[Array[Float]] = { 177 | // cache rdd 178 | rdd.cache() 179 | 180 | // get first entry, to get matrix dimensions 181 | val one = rdd.first 182 | 183 | // build matrix 184 | val matrix = Array.fill[Float](one.getHeight, one.getEnd)(Float.NaN) 185 | 186 | // iterate over rdd and build matrix 187 | rdd.toLocalIterator.foreach(v => { 188 | matrix(v.getXPos)(v.getYPos) = v.getValue 189 | }) 190 | 191 | // unpersist rdd 192 | rdd.unpersist() 193 | 194 | matrix 195 | } 196 | 197 | // create job 198 | val job = newJob(sc) 199 | 200 | //Main program entrance 201 | val flist = new File(src).listFiles.filter(_.getName.endsWith(".fits")) 202 | 203 | // get data and metadata 204 | //val fitsList = (0 until flist.length).map(i => FitsUtils.readFits(flist(i).toString)).toArray 205 | val dflist = sc.parallelize(flist) 206 | val fitsList = dflist.map(f => FitsUtils.readFits(f.toString)).toArray 207 | val map = FitsUtils.processMeta(fitsList).toSeq.sortBy(_._1).map(kv => kv._2) 208 | 209 | // get template 210 | //val template = new Template(tmp) 211 | 212 | // parallelize files 213 | val fitsRdd = buildUp(fitsList.zip(map).toIterator) 214 | 215 | // save to parquet 216 | save(fitsRdd, job, dst) 217 | fitsRdd.unpersist() 218 | 219 | /* 220 | val start = System.currentTimeMillis() 221 | // load back from parquet, just because 222 | ParquetInputFormat.setReadSupportClass(job, classOf[AvroReadSupport[FitsValue]]) 223 | val records = sc.newAPIHadoopFile("resources/tmp_parquet", 224 | classOf[ParquetInputFormat[FitsValue]], 225 | classOf[Void], 226 | classOf[FitsValue], 227 | ContextUtil.getConfiguration(job)).map(p => p._2) 228 | 229 | // reduce down to matrix 230 | val matrixRdd = add(records, template.tcol, template.trow) 231 | 232 | // save to parquet 233 | save(matrixRdd, job, "resources/final_parquet") 234 | matrixRdd.unpersist() 235 | 236 | // again, load back from parquet, just because 237 | ParquetInputFormat.setReadSupportClass(job, classOf[AvroReadSupport[FitsValue]]) 238 | val reloadedMatrixRdd = sc.newAPIHadoopFile("resources/final_parquet", 239 | classOf[ParquetInputFormat[FitsValue]], 240 | classOf[Void], 241 | classOf[FitsValue], 242 | ContextUtil.getConfiguration(job)).map(p => p._2) 243 | 244 | val end = System.currentTimeMillis() 245 | println("madd in parquet takes: " + (end - start).toFloat / 1000) 246 | // build matrix 247 | val matrix = buildMatrix(reloadedMatrixRdd) 248 | 249 | // save output 250 | FitsUtils.createFits(template, matrix, "final.fits")*/ 251 | /*val matrixRdd = add(fitsRdd, template.tcol, template.trow) 252 | val matrix = buildMatrix(matrixRdd) 253 | FitsUtils.createFits(template, matrix, "final.fits")*/ 254 | 255 | /*val template = new Template("resources/template.hdr") 256 | 257 | ParquetInputFormat.setReadSupportClass(job, classOf[AvroReadSupport[FitsValue]]) 258 | val reloadedMatrixRdd = sc.newAPIHadoopFile("resources/final_parquet", 259 | classOf[ParquetInputFormat[FitsValue]], 260 | classOf[Void], 261 | classOf[FitsValue], 262 | ContextUtil.getConfiguration(job)).map(p => p._2) 263 | 264 | val matrix = buildMatrix(reloadedMatrixRdd) 265 | 266 | FitsUtils.createFits(template, matrix, "final.fits")*/ 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/models/Coordinate.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.models 17 | 18 | case class Coordinate(x: Int, y: Int) { 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/models/Fits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.models 17 | 18 | case class Fits(path: String, 19 | naxis: Array[Int], 20 | crpix: Array[Double], 21 | ncol: Int, 22 | matrix: Array[Array[Float]]) { 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/models/FitsMetadata.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.models 17 | 18 | case class FitsMetadata(start: Int, end: Int, offset: Int) { 19 | 20 | val length = end - start + 1 21 | } 22 | 23 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/models/Template.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.models 17 | 18 | import scala.io.Source 19 | 20 | /** 21 | * Class describing the templates for merging images. 22 | * 23 | * @param path The location to populate this from. 24 | */ 25 | class Template(private val path: String) { 26 | var kmap = Source.fromFile(path) 27 | .getLines() 28 | .flatMap(line => { 29 | var words = line.split('=').map(_.trim) 30 | 31 | if (words.length > 1) { 32 | Some((words(0), words(1))) 33 | } else { 34 | None 35 | } 36 | }).toMap 37 | 38 | def tcol = kmap("NAXIS1").toInt 39 | def trow = kmap("NAXIS2").toInt 40 | def crpix: Array[Double] = Array(kmap("CRPIX1").toFloat, kmap("CRPIX2").toFloat) 41 | def crval: Array[Double] = Array(kmap("CRVAL1").toFloat, kmap("CRVAL2").toFloat) 42 | def cdelt: Array[Double] = Array(kmap("CDELT1").toFloat, kmap("CDELT2").toFloat) 43 | def ctype: Array[String] = Array(kmap("CTYPE1"), kmap("CTYPE2")) 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/serialization/MaddKryoRegistrator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.serialization 17 | 18 | import com.esotericsoftware.kryo.Kryo 19 | import com.twitter.chill.avro.AvroSerializer 20 | import edu.berkeley.bids.kira.avro.FitsValue 21 | import org.apache.spark.serializer.KryoRegistrator 22 | import scala.reflect.ClassTag 23 | 24 | class MaddKryoRegistrator extends KryoRegistrator { 25 | override def registerClasses(kryo: Kryo) { 26 | kryo.register(classOf[FitsValue], AvroSerializer.SpecificRecordSerializer[FitsValue]) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/edu/berkeley/bids/kira/util/FitsUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014. Regents of the University of California 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.berkeley.bids.kira.util 17 | 18 | import edu.berkeley.bids.kira.models._ 19 | import org.eso.fits.{ 20 | Fits => ESOFits, 21 | FitsFile, 22 | FitsHDUnit, 23 | FitsHeader, 24 | FitsKeyword, 25 | FitsMatrix, 26 | FitsWCS 27 | } 28 | 29 | object FitsUtils { 30 | 31 | /** 32 | * Extracts FITS data from a file. 33 | * 34 | * @param path Path to the file. 35 | * @return Returns a FITS dataset. 36 | */ 37 | def readFits(path: String): Fits = { 38 | // open file and get header 39 | val file = new FitsFile(path) 40 | val hdu: FitsHDUnit = file.getHDUnit(0) 41 | val hdr = hdu.getHeader() 42 | 43 | // extract matrix data 44 | val dm: FitsMatrix = hdu.getData().asInstanceOf[FitsMatrix] 45 | val naxis = dm.getNaxis() 46 | val crpix = dm.getCrpix() 47 | val ncol = naxis(0) 48 | val nval = dm.getNoValues() 49 | val nrow = nval / ncol 50 | 51 | // build and populate an array 52 | var matrix = Array.ofDim[Float](nrow, ncol) 53 | (0 until nrow).map(i => dm.getFloatValues(i * ncol, ncol, matrix(i))) 54 | 55 | Fits(path, naxis, crpix, ncol, matrix) 56 | } 57 | 58 | /** 59 | * Processes the metadata for all the FITS files. 60 | * 61 | * @param fitsList An array of FITS images. 62 | * @return 63 | */ 64 | def processMeta(fitsList: Array[Fits]): Map[Int, FitsMetadata] = { 65 | // get the max of the second element in the crpix arrays 66 | val crpix0max = fitsList.map(f => f.crpix(0)).max 67 | 68 | // get the max of the second element in the crpix arrays 69 | val crpix1max = fitsList.map(f => f.crpix(1)).max 70 | 71 | fitsList.zipWithIndex.map(vk => { 72 | val (fit, idx) = vk 73 | 74 | // calculate start, end, and offset 75 | val start = crpix1max - fit.crpix(1) + 1 76 | val end = start + fit.naxis(1) - 1 77 | val offset = crpix0max - fit.crpix(0) 78 | 79 | (idx, FitsMetadata(start.toInt, end.toInt, offset.toInt)) 80 | }).toMap 81 | } 82 | 83 | /** 84 | * Saves the output as a Fits file. 85 | * 86 | * @param template Template for saving. 87 | * @param matrix 2D matrix of floats. 88 | * @param path Path to save file at. 89 | */ 90 | def createFits(template: Template, matrix: Array[Array[Float]], path: String) { 91 | // extract the total number of columns and rows 92 | val tcol = template.tcol 93 | val trow = template.trow 94 | 95 | // populate the axis array 96 | val naxis = Array(tcol, trow) 97 | 98 | // flat map the matrix down into a single long array 99 | val data = matrix.flatMap(a => a) 100 | 101 | // allocate header unit and matrix 102 | val mtx: FitsMatrix = new FitsMatrix(ESOFits.DOUBLE, naxis) 103 | 104 | // copy doubles into fits matrix 105 | mtx.setFloatValues(0, data) 106 | 107 | // populate from template 108 | mtx.setCrpix(template.crpix) 109 | mtx.setCrval(template.crval) 110 | mtx.setCdelt(template.cdelt) 111 | 112 | // build header 113 | var hdr: FitsHeader = mtx.getHeader() 114 | hdr.addKeyword(new FitsKeyword("", "")) 115 | var fitsWCS = new FitsWCS(hdr) 116 | hdr.addKeyword(new FitsKeyword("CTYPE1", "= " + template.ctype(0))) 117 | hdr.addKeyword(new FitsKeyword("CTYPE2", "= " + template.ctype(1))) 118 | 119 | // construct hd unit 120 | val hdu = new FitsHDUnit(hdr, mtx) 121 | 122 | // build and write out file 123 | var file: FitsFile = new FitsFile() 124 | file.addHDUnit(hdu) 125 | file.writeFile(path) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /test/scala/README.md: -------------------------------------------------------------------------------- 1 | Kira 2 | ============ 3 | --------------------------------------------------------------------------------