22 | ```
23 |
24 |
--------------------------------------------------------------------------------
/helios/scripts/visualise_ts_rec.R:
--------------------------------------------------------------------------------
1 | #/usr/bin/Rscript
2 | library(ggplot2)
3 | library(reshape2)
4 | library(latex2exp)
5 |
6 | args <- commandArgs(trailingOnly = TRUE)
7 | direc <- args[1]
8 | file <- args[2]
9 | iden <- args[3]
10 |
11 | setwd(direc)
12 |
13 | scatter_df <- read.csv(file, header = FALSE)
14 | colnames(scatter_df) <- c("time", "Value", "Type")
15 |
16 |
17 | palette <- c("#000000", "#CC0C0099", "#5C88DA99")
18 | palette1 <- c("firebrick3", "gray27", "forestgreen")
19 |
20 |
21 | ggplot(scatter_df, aes(x=time, y=Value, color=Type)) +
22 | geom_point(size = 0.75) +
23 | geom_line() +
24 | scale_colour_manual(labels = c("Actual", "Prediction"), values=palette1) +
25 | theme_gray(base_size = 20) +
26 | theme(legend.title=element_blank(), legend.position = "top") +
27 | ylab("km/s") +
28 | xlab(TeX('$t$ (hours)'))
29 |
30 | ggsave(paste(iden, "ts.pdf", sep = ''), scale = 1.0, device = pdf())
--------------------------------------------------------------------------------
/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/WeightedL2FluxLoss.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.helios.core
2 |
3 | import org.platanios.tensorflow.api._
4 | import org.platanios.tensorflow.api.learn.Mode
5 | import org.platanios.tensorflow.api.learn.layers._
6 | import org.platanios.tensorflow.api.ops
7 | import org.platanios.tensorflow.api.ops.Output
8 |
9 | /**
10 | * Weighted L2 Loss Function
11 | * */
12 | case class WeightedL2FluxLoss[
13 | P: TF: IsFloatOrDouble,
14 | T: TF: IsNumeric: IsNotQuantized,
15 | L: TF : IsFloatOrDouble](
16 | override val name: String)
17 | extends Loss[(Output[P], Output[T]), L](name) {
18 |
19 | override val layerType: String = "WeightedL2FluxLoss"
20 |
21 | override def forwardWithoutContext(input: (Output[P], Output[T]))(implicit mode: Mode): Output[L] =
22 | ops.NN.l2Loss((input._1 - input._2.castTo[P])*input._2.castTo[P].sigmoid.sqrt, name).castTo[L]
23 | }
24 |
25 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | *.#*
3 | *#*#
4 | *.swp
5 | *.ipr
6 | *.iml
7 | *.iws
8 | *.pyc
9 | .idea/
10 | .idea_modules/
11 | build/*.jar
12 | .settings
13 | .cache
14 | .cache_data
15 | .DS_Store
16 | release.properties
17 | *.log
18 | data/results*
19 | data/dst_*.txt
20 | _site
21 | data/MagicGamma.csv
22 | data/Adult.csv
23 | data/ForestCover.csv
24 | data/*Res.csv
25 | Gemfile.lock
26 | index*.html
27 | javascripts
28 | stylesheets
29 | data/.RData
30 | data/.Rhistory
31 | rnn-omni-explorer/rsconnect
32 | logs
33 | !project/project/plugins.sbt
34 | project/project
35 | project/target
36 | target
37 | tmp
38 | dist
39 | data/*.png
40 | data/mogp_*.csv
41 | data/*storm*.csv
42 | !data/geomagnetic_storms.csv
43 | !data/geomagnetic_storms2.csv
44 | !data/geomagnetic_storms3.csv
45 | data/osa_training.csv
46 | data/osa_validation.csv
47 | data/all_results.txt
48 | .ensime_cache
49 | .ensime
50 | *result*.csv
51 | .vscode
52 | .metals
53 | .bloop
54 | project/.bloop
55 | project/metals.sbt
56 | .scalafmt.conf
--------------------------------------------------------------------------------
/helios/scripts/run_mex_omni.sc:
--------------------------------------------------------------------------------
1 | import $exec.helios.scripts.mex_omni
2 | import $exec.helios.scripts.csss
3 |
4 | import _root_.io.github.mandar2812.PlasmaML.omni.OMNIData.Quantities._
5 |
6 | mex_omni.dump_omni_mex_data(
7 | 2010,
8 | 2018,
9 | (8, 16),
10 | List(V_SW, /* V_Lat, V_Lon, */ P , B_X, B_Y, B_Z),
11 | home / 'Downloads / "omni_mex_data.json"
12 | )
13 |
14 | val omni_mex_res = mex_omni(
15 | data_file = home / 'Downloads / "omni_mex_data.json",
16 | start_year = 2015,
17 | end_year = 2017,
18 | test_year = 2016,
19 | network_size = Seq(20, 20),
20 | activation_func = (i: Int) => tf.learn.Sigmoid(s"Act_$i"),//timelag.utils.getSinAct[Double](1, i),
21 | iterations = 100000,
22 | iterations_tuning = 20000,
23 | pdt_iterations_tuning = 4,
24 | pdt_iterations_test = 9,
25 | batch_size = 32,
26 | optimizer = tf.train.Adam(0.001f)
27 | )
28 |
29 | helios.visualise_cdt_results(
30 | csss.scatter_plots_test(omni_mex_res.results.summary_dir).last
31 | )
32 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/VariableValuesRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | /**
4 | * Field data for CDF record of type Variable Values Record.
5 | *
6 | * @author Mark Taylor
7 | * @since 19 Jun 2013
8 | */
9 | public class VariableValuesRecord extends Record {
10 |
11 | private final long recordsOffset_;
12 |
13 | /**
14 | * Constructor.
15 | *
16 | * @param plan basic record information
17 | */
18 | public VariableValuesRecord( RecordPlan plan ) {
19 | super( plan, "VVR", 7 );
20 | Pointer ptr = plan.createContentPointer();
21 | recordsOffset_ = ptr.get();
22 | }
23 |
24 | /**
25 | * Returns the file offset at which the records data in this record
26 | * starts.
27 | *
28 | * @return file offset for start of Records field
29 | */
30 | public long getRecordsOffset() {
31 | return recordsOffset_;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/copy-docs.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1
4 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/omni
5 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/mag-core
6 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/helios
7 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/streamer
8 | mkdir -p ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/vanAllen
9 |
10 | sbt stage
11 |
12 | cp -R omni/target/scala-2.12/api/* ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/omni/
13 | cp -R mag-core/target/scala-2.12/api/* ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/mag-core/
14 | cp -R helios/target/scala-2.12/api/* ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/helios/
15 | cp -R streamer/target/scala-2.12/api/* ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/streamer/
16 | cp -R vanAllen/target/scala-2.12/api/* ../transcendent-ai-labs.github.io/api_docs/PlasmaML/$1/vanAllen/
17 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/CdfFormatException.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | import java.io.IOException;
4 |
5 | /**
6 | * Exception thrown during CDF parsing when the data stream appears either
7 | * to be in contravention of the CDF format, or uses some feature of
8 | * the CDF format which is unsupported by the current implementation.
9 | *
10 | * @author Mark Taylor
11 | * @since 18 Jun 2013
12 | */
13 | public class CdfFormatException extends IOException {
14 |
15 | /**
16 | * Constructs an exception with a message.
17 | *
18 | * @param msg message
19 | */
20 | public CdfFormatException( String msg ) {
21 | super( msg );
22 | }
23 |
24 | /**
25 | * Constructs an exception with a message and a cause.
26 | *
27 | * @param msg message
28 | * @param cause upstream exception
29 | */
30 | public CdfFormatException( String msg, Throwable cause ) {
31 | super( msg );
32 | initCause( cause );
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/helios/scripts/visualise_tl.R:
--------------------------------------------------------------------------------
1 | #/usr/bin/Rscript
2 | library(ggplot2)
3 | library(reshape2)
4 | library(latex2exp)
5 |
6 | args <- commandArgs(trailingOnly = TRUE)
7 | direc <- args[1]
8 | file <- args[2]
9 | iden <- args[3]
10 |
11 | setwd(direc)
12 |
13 | scatter_df <- read.csv(file, header = FALSE)
14 | colnames(scatter_df) <- c("Prediction", "Actual", "TimeLag")
15 |
16 | palette <- c("#000000", "#CC0C0099", "#5C88DA99")
17 |
18 | ggplot(scatter_df, aes(x=Actual, y=Prediction)) +
19 | theme_gray(base_size = 20) +
20 | geom_point(alpha=0.3) +
21 | stat_density_2d(colour = "blue", size=0.65, alpha = 0.85) +
22 | geom_abline(slope = 1, intercept = 0, color = "#CC0C0099", size=1.1, alpha=1) +
23 | ylab(TeX('$\\hat{v}$ (km/s)')) +
24 | xlab(TeX('$v$ (km/s)'))
25 |
26 |
27 | ggsave(paste(iden, "scatter_v.pdf", sep = ''), scale = 1.0, device = pdf())
28 |
29 |
30 |
31 | ggplot(scatter_df, aes(x=Prediction, y=TimeLag)) +
32 | theme_gray(base_size = 20) +
33 | geom_point(alpha=1/3) +
34 | xlab(TeX('$\\hat{v}$ (km/s)')) +
35 | ylab(TeX('$arg\\,max_{i} \ \ \\left(\\hat{p}_{i}\\right)$ (hr)'))
36 |
37 | ggsave(paste(iden, "scatter_v_tl.pdf", sep = ''), scale = 1.0, device = pdf())
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/CdfField.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.record.Record;
4 |
5 | import java.lang.annotation.Documented;
6 | import java.lang.annotation.ElementType;
7 | import java.lang.annotation.Retention;
8 | import java.lang.annotation.RetentionPolicy;
9 | import java.lang.annotation.Target;
10 |
11 | /**
12 | * Marks field members of {@link Record} subclasses which correspond directly
13 | * to fields in typed CDF records in a CDF file.
14 | *
15 | * These fields are all public and final, and have names matching
16 | * (apart perhaps from minor case tweaking)
17 | * the fields documented in the relevant subsections of Section 2 of the
18 | * CDF Internal Format Description document.
19 | *
20 | *
See that document for a description of the meaning of these fields.
21 | *
22 | * @author Mark Taylor
23 | * @since 25 Jun 2013
24 | * @see
25 | * CDF Internal Format Description document
27 | */
28 | @Documented
29 | @Retention(RetentionPolicy.RUNTIME)
30 | @Target(ElementType.FIELD)
31 | public @interface CdfField {
32 | }
33 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/Pointer.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | /**
4 | * Keeps track of a file offset.
5 | *
6 | * @author Mark Taylor
7 | * @since 18 Jun 2013
8 | */
9 | public class Pointer {
10 |
11 | private long value_;
12 |
13 | /**
14 | * Constructor.
15 | *
16 | * @param value initial value
17 | */
18 | public Pointer( long value ) {
19 | value_ = value;
20 | }
21 |
22 | /**
23 | * Returns this pointer's current value.
24 | *
25 | * @return value
26 | */
27 | public long get() {
28 | return value_;
29 | }
30 |
31 | /**
32 | * Returns this pointer's current value and increments it by a given step.
33 | *
34 | * @param increment amount to increase value by
35 | * @return pre-increment value
36 | */
37 | public long getAndIncrement( int increment ) {
38 | long v = value_;
39 | value_ += increment;
40 | return v;
41 | }
42 |
43 | /**
44 | * Sets this pointer's current value.
45 | *
46 | * @param value new value
47 | */
48 | public void set( long value ) {
49 | value_ = value;
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/CompressedParametersRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 |
6 | import java.io.IOException;
7 |
8 | /**
9 | * Field data for CDF record of type Compressed Parameters Record.
10 | *
11 | * @author Mark Taylor
12 | * @since 19 Jun 2013
13 | */
14 | public class CompressedParametersRecord extends Record {
15 |
16 | @CdfField
17 | public final int cType;
18 | @CdfField public final int rfuA;
19 | @CdfField public final int pCount;
20 | @CdfField public final int[] cParms;
21 |
22 | /**
23 | * Constructor.
24 | *
25 | * @param plan basic record information
26 | */
27 | public CompressedParametersRecord( RecordPlan plan ) throws IOException {
28 | super( plan, "CPR", 11 );
29 | Buf buf = plan.getBuf();
30 | Pointer ptr = plan.createContentPointer();
31 | this.cType = buf.readInt( ptr );
32 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
33 | this.pCount = buf.readInt( ptr );
34 | this.cParms = readIntArray( buf, ptr, this.pCount );
35 | checkEndRecord( ptr );
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/helios/scripts/csss_bs_job.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.{utils => dutils}
2 | import $file.csss
3 | import $file.csss_pdt_model_tuning
4 | import $file.csss_so_tuning
5 | import $file.env
6 | import _root_.io.github.mandar2812.dynaml.repl.Router.main
7 | import org.joda.time._
8 | import ammonite.ops._
9 | import ammonite.ops.ImplicitWd._
10 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
11 | import _root_.io.github.mandar2812.PlasmaML.omni.OMNIData
12 |
13 | @main
14 | def main(
15 | csss_job_id: String,
16 | exp_dir: String,
17 | network_size: Seq[Int] = Seq(50, 50),
18 | root_dir: String = env.summary_dir.toString
19 | ) = {
20 |
21 | val csss_fixed = csss_so_tuning.baseline(
22 | Path(s"${root_dir}/${csss_job_id}/${exp_dir}"),
23 | network_size = network_size,
24 | activation_func = (i: Int) => timelag.utils.getReLUAct3[Double](1, 1, i, 0f),
25 | optimization_algo = org.platanios.tensorflow.api.tf.train.Adam(0.01f),
26 | max_iterations = csss.ext_iterations,
27 | max_iterations_tuning = csss.base_iterations,
28 | batch_size = 128,
29 | num_samples = 4,
30 | data_scaling = "hybrid",
31 | use_copula = true
32 | )
33 |
34 | println("Base Line Model Performance:")
35 | csss_fixed.results.metrics_test.get.print()
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/SparsenessParametersRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 |
6 | import java.io.IOException;
7 |
8 | /**
9 | * Field data for CDF record of type Sparseness Parameters Record.
10 | *
11 | * @author Mark Taylor
12 | * @since 19 Jun 2013
13 | */
14 | public class SparsenessParametersRecord extends Record {
15 |
16 | @CdfField
17 | public final int sArraysType;
18 | @CdfField public final int rfuA;
19 | @CdfField public final int pCount;
20 | @CdfField public final int[] sArraysParms;
21 |
22 | /**
23 | * Constructor.
24 | *
25 | * @param plan basic record information
26 | */
27 | public SparsenessParametersRecord( RecordPlan plan ) throws IOException {
28 | super( plan, "SPR", 12 );
29 | Buf buf = plan.getBuf();
30 | Pointer ptr = plan.createContentPointer();
31 | this.sArraysType = buf.readInt( ptr );
32 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
33 | this.pCount = buf.readInt( ptr );
34 | this.sArraysParms = readIntArray( buf, ptr, this.pCount );
35 | checkEndRecord( ptr );
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/GlobalAttribute.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | /**
4 | * Provides the description and entry values
5 | * for CDF attribute with global scope.
6 | *
7 | *
The gEntries and zEntries are combined in a single list,
8 | * on the grounds that users are not likely to be much interested
9 | * in the difference.
10 | *
11 | * @author Mark Taylor
12 | * @since 20 Jun 2013
13 | */
14 | public class GlobalAttribute {
15 |
16 | private final String name_;
17 | private final AttributeEntry[] entries_;
18 |
19 | /**
20 | * Constructor.
21 | *
22 | * @param name attribute name
23 | * @param entries attribute entries
24 | */
25 | public GlobalAttribute( String name, AttributeEntry[] entries ) {
26 | name_ = name;
27 | entries_ = entries;
28 | }
29 |
30 | /**
31 | * Returns this attribute's name.
32 | *
33 | * @return attribute name
34 | */
35 | public String getName() {
36 | return name_;
37 | }
38 |
39 | /**
40 | * Returns this attribute's entry values.
41 | *
42 | * @return entry values for this attribute
43 | */
44 | public AttributeEntry[] getEntries() {
45 | return entries_;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/streamer/scripts/streamerShowerTest.scala:
--------------------------------------------------------------------------------
1 | import breeze.linalg.DenseVector
2 | import io.github.mandar2812.PlasmaML.streamer.StreamerShowerEmulator
3 | import io.github.mandar2812.dynaml.kernels._
4 |
5 | import io.github.mandar2812.dynaml.analysis.VectorField
6 |
7 | val num_features = 5
8 | implicit val ev = VectorField(num_features)
9 |
10 |
11 | val stK = new TStudentKernel(2.2)
12 | val sqExpK = new SEKernel(1.5, 0.7)
13 | val tKernel = new TStudentKernel(0.5+1.0/num_features)
14 | tKernel.block_all_hyper_parameters
15 | val mlpKernel = new MLPKernel(10.0/num_features.toDouble, 1.382083995440671)
16 |
17 | sqExpK.block("amplitude")
18 | val linearK = new PolynomialKernel(1, 0.0)
19 | linearK.block_all_hyper_parameters
20 |
21 | val perKernel = new PeriodicKernel(4.5, 2.5)
22 |
23 | val cauK = new CauchyKernel(2.1)
24 | cauK.block_all_hyper_parameters
25 |
26 | val n = new DiracKernel(0.01)
27 | n.block_all_hyper_parameters
28 |
29 | StreamerShowerEmulator.globalOpt = "GS"
30 |
31 | StreamerShowerEmulator.trainingSize = 2000
32 | StreamerShowerEmulator.testSize = 2000
33 |
34 | val resStreamerSTP = StreamerShowerEmulator(linearK + mlpKernel + tKernel, n, 4.0, 2, 0.2, false, 15)
35 |
36 | resStreamerSTP.print
37 |
38 | val resStreamerGP = StreamerShowerEmulator(linearK + mlpKernel + tKernel, n, 3, 0.2, false, 25)
39 |
40 | resStreamerGP.print
41 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/utils/package.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML
2 |
3 | /**
4 | * Miscellaneous utilities for the PlasmaML software distribution.
5 | *
6 | * @author mandar2812 date 16/05/2017.
7 | * */
8 | package object utils {
9 |
10 | def mean(seq: Seq[Double]): Double = seq.sum/seq.length
11 |
12 | def generalized_logistic(
13 | lower: Double, upper: Double, rate: Double,
14 | start_time: Double, µ: Double, q: Double,
15 | c: Double)(t: Double) = {
16 | require(µ > 0d, "In a generalized logistic curve, growth order must be positive")
17 | lower + (upper - lower)/math.pow(c + q*math.exp(-rate*(t - start_time)), 1d/µ)
18 | }
19 |
20 |
21 | /**
22 | * Calculate the generalised Laguerre polynomial
23 | * */
24 | def laguerre(n: Int, alpha: Double, x: Double): Double = {
25 |
26 | def laguerreRec(k: Int, alphav: Double, xv: Double, a: Double, b: Double): Double = k match {
27 | case 0 => a
28 | case 1 => b
29 | case _ => laguerreRec(k - 1, xv, alphav, ((2*k + 1 + alphav - xv)*a - (k + alphav)*b)/(k + 1), a)
30 | }
31 |
32 | laguerreRec(n, alpha, x, 1.0, 1.0 + alpha - x)
33 | }
34 |
35 | def grad_laguerre(k: Int)(n: Int, alpha: Double, x: Double): Double =
36 | if(k <= n) math.pow(-1.0, k)*laguerre(n - k, alpha + k, x)
37 | else 0d
38 | }
39 |
--------------------------------------------------------------------------------
/helios/scripts/download_sdo.sc:
--------------------------------------------------------------------------------
1 | import ammonite.ops._
2 | import org.joda.time._
3 | import io.github.mandar2812.PlasmaML.helios
4 | import io.github.mandar2812.PlasmaML.helios.data.{SDO, SDOData}
5 | import io.github.mandar2812.PlasmaML.helios.data.SDOData.Instruments._
6 | import io.github.mandar2812.PlasmaML.helios.data.SDOData.Resolutions._
7 | import io.github.mandar2812.dynaml.repl.Router.main
8 | import org.joda.time.format.DateTimeFormat
9 |
10 | DateTimeZone.setDefault(DateTimeZone.UTC)
11 |
12 | /**
13 | * Perform a bulk download of the SOHO archive.
14 | *
15 | * @param start_date Download starting from a date string "yyyy-mm-dd"
16 | * @param end_date Download until a date string "yyyy-mm-dd"
17 | * @param path Destination directory to put data
18 | * @param size Resolution of the images to be downloaded, 512 or 1024
19 | * @param instrument The SDO instrument from which to download,
20 | * specified as a string. See [[SDOData.Instruments]]
21 | * */
22 | @main
23 | def apply(start_date: String, end_date: String, path: Path, size: Int, instrument: String): Unit = {
24 | val formatter = DateTimeFormat.forPattern("YYYY-MM-dd")
25 |
26 | val start_dt = formatter.parseLocalDate(start_date)
27 | val end_dt = formatter.parseLocalDate(end_date)
28 |
29 | helios.data.download_image_data(SDO(instrument, size), path)(start_dt, end_dt)
30 | }
31 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/utils/DiracTuple2Kernel.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.utils
2 |
3 | import breeze.linalg.{DenseMatrix, DenseVector}
4 | import io.github.mandar2812.dynaml.kernels.{KernelMatrix, LocalSVMKernel, SVMKernelMatrix}
5 |
6 | class DiracTuple2Kernel(private var noiseLevel: Double = 1.0)
7 | extends LocalSVMKernel[(Double, Double)]
8 | with Serializable {
9 |
10 | override val hyper_parameters = List("noiseLevel")
11 |
12 | state = Map("noiseLevel" -> noiseLevel)
13 |
14 | def setNoiseLevel(d: Double): Unit = {
15 | this.state += ("noiseLevel" -> d)
16 | this.noiseLevel = d
17 | }
18 |
19 | override def evaluateAt(
20 | config: Map[String, Double])(
21 | x: (Double, Double), y: (Double, Double)): Double =
22 | if (x._1 == y._1 && x._2 == y._2) math.abs(config("noiseLevel"))*1.0 else 0.0
23 |
24 | override def gradientAt(
25 | config: Map[String, Double])(
26 | x: (Double, Double), y: (Double, Double)): Map[String, Double] =
27 | Map("noiseLevel" -> 1.0*evaluateAt(config)(x,y)/math.abs(config("noiseLevel")))
28 |
29 | override def buildKernelMatrix[S <: Seq[(Double, Double)]](
30 | mappedData: S, length: Int): KernelMatrix[DenseMatrix[Double]] =
31 | new SVMKernelMatrix(
32 | DenseMatrix.eye[Double](length)*state("noiseLevel"),
33 | length)
34 |
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/helios/scripts/download_soho.sc:
--------------------------------------------------------------------------------
1 | import ammonite.ops._
2 | import org.joda.time._
3 | import io.github.mandar2812.PlasmaML.helios
4 | import io.github.mandar2812.PlasmaML.helios.data.{SOHO, SOHOData}
5 | import io.github.mandar2812.PlasmaML.helios.data.SOHOData.Instruments._
6 | import io.github.mandar2812.PlasmaML.helios.data.SOHOData.Resolutions._
7 | import io.github.mandar2812.dynaml.repl.Router.main
8 | import org.joda.time.format.DateTimeFormat
9 |
10 | DateTimeZone.setDefault(DateTimeZone.UTC)
11 |
12 | /**
13 | * Perform a bulk download of the SOHO archive.
14 | *
15 | * @param start_date Download starting from a date string "yyyy-mm-dd"
16 | * @param end_date Download until a date string "yyyy-mm-dd"
17 | * @param path Destination directory to put data
18 | * @param size Resolution of the images to be downloaded, 512 or 1024
19 | * @param instrument The SOHO instrument from which to download,
20 | * specified as a string. See [[SOHOData.Instruments]]
21 | * */
22 | @main
23 | def apply(start_date: String, end_date: String, path: Path, size: Int, instrument: String): Unit = {
24 | val formatter = DateTimeFormat.forPattern("YYYY-MM-dd")
25 |
26 | val start_dt = formatter.parseLocalDate(start_date)
27 | val end_dt = formatter.parseLocalDate(end_date)
28 |
29 | helios.data.download_image_data(SOHO(instrument, size), path)(start_dt, end_dt)
30 | }
31 |
--------------------------------------------------------------------------------
/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/AutoEncoder.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.helios.core
2 |
3 | import org.platanios.tensorflow.api.learn.Mode
4 | import org.platanios.tensorflow.api.learn.layers.Layer
5 |
6 | /**
7 | *
Compression & Representation
8 | *
9 | * A layer which consists of compression and reconstruction modules.
10 | * Useful for unsupervised learning tasks.
11 | * */
12 | case class AutoEncoder[I, J](
13 | override val name: String,
14 | encoder: Layer[I, J],
15 | decoder: Layer[J, I]) extends
16 | Layer[I, (J, I)](name) {
17 |
18 | self =>
19 |
20 | override val layerType: String = s"Representation[${encoder.layerType}, ${decoder.layerType}]"
21 |
22 | override def forwardWithoutContext(input: I)(implicit mode: Mode): (J, I) = {
23 |
24 | val encoding = encoder(input)
25 |
26 | val reconstruction = decoder(encoding)
27 |
28 | (encoding, reconstruction)
29 | }
30 |
31 | def >>[S](other: AutoEncoder[J, S]): ComposedAutoEncoder[I, J, S] = new ComposedAutoEncoder(name, self, other)
32 | }
33 |
34 | class ComposedAutoEncoder[I, J, K](
35 | override val name: String,
36 | encoder1: AutoEncoder[I, J],
37 | encoder2: AutoEncoder[J, K]) extends
38 | AutoEncoder[I, K](
39 | s"Compose[${encoder1.name}, ${encoder2.name}]",
40 | encoder1.encoder >> encoder2.encoder,
41 | encoder2.decoder >> encoder1.decoder)
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/CompressedVariableValuesRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 |
6 | import java.io.IOException;
7 |
8 | /**
9 | * Field data for CDF record of type Compressed Variable Values Record.
10 | *
11 | * @author Mark Taylor
12 | * @since 19 Jun 2013
13 | */
14 | public class CompressedVariableValuesRecord extends Record {
15 |
16 | @CdfField
17 | public final int rfuA;
18 | @CdfField public final long cSize;
19 | private final long dataOffset_;
20 |
21 | /**
22 | * Constructor.
23 | *
24 | * @param plan basic record information
25 | */
26 | public CompressedVariableValuesRecord( RecordPlan plan )
27 | throws IOException {
28 | super( plan, "CVVR", 13 );
29 | Buf buf = plan.getBuf();
30 | Pointer ptr = plan.createContentPointer();
31 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
32 | this.cSize = buf.readOffset( ptr );
33 | dataOffset_ = ptr.get();
34 | }
35 |
36 | /**
37 | * Returns the file offset at which the compressed data in
38 | * this record starts.
39 | *
40 | * @return file offset for start of data field
41 | */
42 | public long getDataOffset() {
43 | return dataOffset_;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/omni/scripts/gpImagePriors.sc:
--------------------------------------------------------------------------------
1 | import breeze.linalg.DenseVector
2 | import com.sksamuel.scrimage._
3 | import io.github.mandar2812.dynaml.analysis.VectorField
4 | import io.github.mandar2812.dynaml.kernels.{DiracKernel, GaussianSpectralKernel, MAKernel}
5 | import io.github.mandar2812.dynaml.models.bayes.LinearTrendGaussianPrior
6 |
7 | import scala.collection.mutable.{MutableList => MutList}
8 |
9 | //Load the image into an object, and scale it by 0.25
10 | val im = Image.fromFile(new java.io.File("../../tmp/20011030_0135_mdimag_512.jpg")).scale(0.0625)
11 |
12 |
13 | val scale = 512*0.0625
14 | val imageData: MutList[(DenseVector[Double], Double)] = MutList()
15 |
16 | im.foreach((x, y, p) => {
17 | val coordinates: DenseVector[Double] = DenseVector(x/scale, y/scale)
18 | val pixelInt = PixelTools.gray(p.toARGBInt)
19 | imageData += ((coordinates, pixelInt))
20 | })
21 |
22 | val encoder = GaussianSpectralKernel.getEncoderforBreezeDV(2)
23 | implicit val field = VectorField(2)
24 |
25 | val n = new DiracKernel(2.5)
26 | val n1 = new MAKernel(1.5)
27 |
28 | val gsmKernel = GaussianSpectralKernel[DenseVector[Double]](
29 | DenseVector(0.5, 0.5), DenseVector(1.0, 1.0),
30 | encoder)
31 |
32 |
33 | val gp_prior = new LinearTrendGaussianPrior[DenseVector[Double]](gsmKernel, n, DenseVector.zeros[Double](2), 128.0)
34 |
35 |
36 | gp_prior.globalOptConfig_(Map("gridStep" -> "0.15", "gridSize" -> "2", "globalOpt" -> "GPC", "policy" -> "GS"))
37 | val gpModel = gp_prior.posteriorModel(imageData)
38 |
39 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/DiffusionPrior.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import io.github.mandar2812.dynaml.kernels.{LocalScalarKernel, MAKernel}
4 | import io.github.mandar2812.dynaml.models.bayes.CoRegGPPrior
5 | import io.github.mandar2812.dynaml.pipes.Encoder
6 | import io.github.mandar2812.dynaml.DynaMLPipe._
7 |
8 | /**
9 | * Extension of [[io.github.mandar2812.dynaml.models.bayes.CoRegGPPrior]], adapted
10 | * for the plasma diffusion scheme.
11 | *
12 | * @author mandar2812 date 27/06/2017.
13 | * */
14 | class DiffusionPrior(
15 | val trend: MagnetosphericProcessTrend[Map[String, Double]],
16 | val covarianceSpace: LocalScalarKernel[Double],
17 | val covarianceTime: LocalScalarKernel[Double],
18 | val noise: Double = 0.5,
19 | initialParams: (Double, Double, Double, Double) = (1.0*math.pow(10d, 0.506), 10d, 0d, -9.325)) extends
20 | CoRegGPPrior[Double, Double, Map[String, Double]](
21 | covarianceSpace, covarianceTime,
22 | new MAKernel(noise), new MAKernel(noise),
23 | Encoder(identityPipe[Map[String, Double]], identityPipe[Map[String, Double]])) {
24 |
25 |
26 | protected var parameters: (Double, Double, Double, Double) = initialParams
27 |
28 | override def _meanFuncParams = trend.transform.i(parameters)
29 |
30 | override def meanFuncParams_(p: Map[String, Double]) = {
31 | parameters = trend.transform(p)
32 | }
33 |
34 | override val meanFunctionPipe = trend
35 | }
36 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/CompressedCdfRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type Compressed CDF Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class CompressedCdfRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long cprOffset;
20 | @CdfField public final long uSize;
21 | @CdfField public final int rfuA;
22 | private final long dataOffset_;
23 |
24 | /**
25 | * Constructor.
26 | *
27 | * @param plan basic record information
28 | */
29 | public CompressedCdfRecord( RecordPlan plan ) throws IOException {
30 | super( plan, "CCR", 10 );
31 | Buf buf = plan.getBuf();
32 | Pointer ptr = plan.createContentPointer();
33 | this.cprOffset = buf.readOffset( ptr );
34 | this.uSize = buf.readOffset( ptr );
35 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
36 | dataOffset_ = ptr.get();
37 | }
38 |
39 | /**
40 | * Returns the file offset at which the compressed data in
41 | * this record starts.
42 | *
43 | * @return file offset for start of data field
44 | */
45 | public long getDataOffset() {
46 | return dataOffset_;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/VariableIndexRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type Variable Index Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class VariableIndexRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long vxrNext;
20 | @CdfField public final int nEntries;
21 | @CdfField public final int nUsedEntries;
22 | @CdfField public final int[] first;
23 | @CdfField public final int[] last;
24 | @CdfField @OffsetField public final long[] offset;
25 |
26 | /**
27 | * Constructor.
28 | *
29 | * @param plan basic record information
30 | */
31 | public VariableIndexRecord( RecordPlan plan ) throws IOException {
32 | super( plan, "VXR", 6 );
33 | Buf buf = plan.getBuf();
34 | Pointer ptr = plan.createContentPointer();
35 | this.vxrNext = buf.readOffset( ptr );
36 | this.nEntries = buf.readInt( ptr );
37 | this.nUsedEntries = buf.readInt( ptr );
38 | this.first = readIntArray( buf, ptr, this.nEntries );
39 | this.last = readIntArray( buf, ptr, this.nEntries );
40 | this.offset = readOffsetArray( buf, ptr, this.nEntries );
41 | checkEndRecord( ptr );
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/omni/scripts/loadData.R:
--------------------------------------------------------------------------------
1 | #! /usr/bin/Rscript
2 | args <- commandArgs(trailingOnly = TRUE)
3 | year <- args[1]
4 | setwd("../../data/")
5 | prefix <- "omni2_"
6 | system(paste("wget ftp://spdf.gsfc.nasa.gov/pub/data/omni/low_res_omni/",
7 | prefix, year, ".dat", sep = ""))
8 | system(paste("sed -i 's/ \\{1,\\}/,/g' ", prefix, year, ".dat", sep = ""))
9 |
10 |
11 | df <- read.csv(paste(prefix, year, ".dat", sep = ""),
12 | header = FALSE, stringsAsFactors = FALSE,
13 | colClasses = rep("numeric",55),
14 | na.strings = c("99", "999.9",
15 | "9999.", "9.999", "99.99",
16 | "9999", "999999.99",
17 | "99999.99", "9999999."))
18 |
19 | library(MonetDB.R)
20 | bat.file <- monetdb.server.setup(paste(getwd(), "monetdb", sep="/"),
21 | monetdb.program.path = "~/monetdb/bin/",
22 | dbname = "omni", dbport = 50000)
23 | correct.pid <- monetdb.server.start(bat.file)
24 |
25 | conn <- dbConnect(MonetDB.R(), "monetdb://localhost:50000/omni")
26 |
27 | monetdb.read.csv(conn, paste(prefix, year, ".dat", sep = ""), "omniHourly",
28 | nrows=8760, header=FALSE,
29 | locked=FALSE,
30 | na.strings=c("99", "999.9",
31 | "9999.", "9.999", "99.99",
32 | "9999", "999999.99",
33 | "99999.99", "9999999."),
34 | nrow.check=500, delim=",",
35 | newline = "\\n", quote = "\"",
36 | create=TRUE)
37 |
38 | monetdb.server.stop(correct.pid, wait = TRUE)
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/package.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.dynaml.pipes.DataPipe2
5 |
6 | /**
7 | * Primitives for diffusion dynamics.
8 | *
9 | * @author mandar2812 date 10/07/2017.
10 | * */
11 | package object diffusion {
12 |
13 | type breezeVec = DenseVector[Double]
14 |
15 | val sqNormBDV = DataPipe2((x: breezeVec, y: breezeVec) => {
16 | val d = x-y
17 | d dot d
18 | })
19 |
20 | val sqNormDouble = DataPipe2((x: Double, y: Double) => {
21 | math.pow(x-y, 2.0)
22 | })
23 |
24 | val l1NormDouble = DataPipe2((x: Double, y: Double) => {
25 | math.abs(x-y)
26 | })
27 |
28 | def gradSqNormDouble(order_x: Int, order_y: Int)(x: Double, y: Double): Double = {
29 | require(
30 | order_x >= 0 && order_y >= 0,
31 | "Orders of differentiation must be non negative!!")
32 |
33 | val order = order_x + order_y
34 | if(order > 2) 0d
35 | else if (order == 0) sqNormDouble(x,y)
36 | else (order_x, order_y) match {
37 | case (1, 0) => 2d*math.abs(x-y)
38 | case (0, 1) => -2d*math.abs(x-y)
39 | case (1, 1) => -2d
40 | case _ => 2d
41 | }
42 | }
43 |
44 | def gradL1NormDouble(order_x: Int, order_y: Int)(x: Double, y: Double): Double = {
45 | require(
46 | order_x >= 0 && order_y >= 0,
47 | "Orders of differentiation must be non negative!!")
48 |
49 | val order = order_x + order_y
50 | if(order > 1) 0d
51 | else if (order == 0) l1NormDouble(x,y)
52 | else (order_x, order_y) match {
53 | case (1, 0) => 1d
54 | case (0, 1) => 1d
55 | case _ => 1d
56 | }
57 | }
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/helios/scripts/read_fits_data.sc:
--------------------------------------------------------------------------------
1 | import nom.tam.fits._
2 | import _root_.io.github.mandar2812.PlasmaML.helios.fte
3 | import _root_.io.github.mandar2812.dynaml.tensorflow._
4 | import _root_.io.github.mandar2812.dynaml.{utils => dutils}
5 |
6 | //Some hard-coded meta data for FTE/Bss files.
7 |
8 | //The latitude discretization
9 | val latitude_grid = {
10 | dutils
11 | .range[Double](-1d, 1d, 360)
12 | .map(math.asin)
13 | .map(math.toDegrees)
14 | .zipWithIndex
15 | .filter(c => c._2 > 0 && c._2 <= 359 && c._2 % 2 == 1)
16 | .map(_._1)
17 | .map(x => BigDecimal.binary(x, new java.math.MathContext(4)).toDouble)
18 | }
19 |
20 | //The longitude discretization
21 | val longitude_grid = (1 to 360).map(_.toDouble)
22 |
23 | val files = dtfdata.dataset {
24 | List(
25 | //root / 'Users / 'mandar / 'Downloads / "GONGfte_csss2205HR.fits",
26 | root / 'Users / 'mandar / 'Downloads / "GONGbr1_csss2205HR.fits",
27 | root / 'Users / 'mandar / 'Downloads / "GONGbrcp_csss2205HR.fits",
28 | root / 'Users / 'mandar / 'Downloads / "GONGbrss_csss2205HR.fits"
29 | )
30 | }
31 |
32 | val file_to_array = DataPipe((file: Path) => {
33 | new Fits(file.toString)
34 | .getHDU(0)
35 | .getKernel()
36 | .asInstanceOf[Array[Array[Float]]]
37 | .toIterable
38 | })
39 |
40 | val array_to_patt = DataPipe(
41 | (arr: Iterable[Array[Float]]) =>
42 | latitude_grid
43 | .zip(arr)
44 | .flatMap(
45 | (s: (Double, Array[Float])) =>
46 | longitude_grid
47 | .zip(s._2)
48 | .map(
49 | (p: (Double, Float)) =>
50 | HelioPattern(p._1, s._1, Some(p._2.toDouble))
51 | )
52 | )
53 | .toIterable
54 | )
55 |
56 | files.flatMap(file_to_array > array_to_patt)
57 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/UnusedInternalRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type Unused Internal Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class UnusedInternalRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long nextUir;
20 | @CdfField @OffsetField public final long prevUir;
21 |
22 | /**
23 | * Constructor.
24 | *
25 | * @param plan basic record information
26 | */
27 | public UnusedInternalRecord( RecordPlan plan ) throws IOException {
28 | super( plan, "UIR", -1 );
29 | Buf buf = plan.getBuf();
30 | Pointer ptr = plan.createContentPointer();
31 | int planHeaderSize = (int) plan.getReadCount( ptr );
32 |
33 | // This UIR may be unsociable and too small to contain UIR fields.
34 | // If so, don't attempt to read them (if we are extremely unlucky
35 | // they might be off the end of the file). Check the record size
36 | // is large enough to accommodate these fields before reading them.
37 | int pointerSize = buf.isBit64() ? 8 : 4;
38 | int sociableUirSize = planHeaderSize + 2 * pointerSize;
39 | if ( plan.getRecordSize() >= sociableUirSize ) {
40 | this.nextUir = buf.readOffset( ptr );
41 | this.prevUir = buf.readOffset( ptr );
42 | }
43 | else { // too small to be sociable
44 | this.nextUir = -1L;
45 | this.prevUir = -1L;
46 | }
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/MagParamBasis.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.PlasmaML.utils.{grad_laguerre, laguerre}
5 | import io.github.mandar2812.dynaml.analysis.DifferentiableMap
6 | import io.github.mandar2812.dynaml.pipes.Basis
7 | import io.github.mandar2812.dynaml.{analysis, utils}
8 |
9 | abstract class MagParamBasis extends
10 | Basis[Double] with
11 | DifferentiableMap[Double, DenseVector[Double], DenseVector[Double]]
12 |
13 |
14 | object MagParamBasis {
15 |
16 | def apply(func: Double => DenseVector[Double], grad_func: Double => DenseVector[Double]): MagParamBasis =
17 | new MagParamBasis {
18 |
19 | override protected val f: Double => DenseVector[Double] = func
20 |
21 | override def J(x: Double): DenseVector[Double] = grad_func(x)
22 | }
23 |
24 | val polynomial_basis: Int => MagParamBasis = n => apply(
25 | analysis.PolynomialBasisGenerator(n).run _,
26 | x => DenseVector.tabulate(n + 1)(i => i * math.pow(x, i - 1))
27 | )
28 |
29 | val chebyshev_basis: Int => MagParamBasis = n => apply(
30 | analysis.ChebyshevBasisGenerator(n, 1).run _,
31 | x => DenseVector(Array(0d) ++ Array.tabulate(n)(i => if(i+1 > 0) (i+1)*utils.chebyshev(i, x, kind = 2) else 0d))
32 | )
33 |
34 | val hermite_basis: Int => MagParamBasis = n => apply(
35 | analysis.HermiteBasisGenerator(n).run _,
36 | x => DenseVector(Array(0d) ++ (1 to n).toArray.map(i => i*utils.hermite(i - 1, x)))
37 | )
38 |
39 | val laguerre_basis: (Int, Double) => MagParamBasis = (n, alpha) => apply(
40 | (x: Double) => DenseVector((0 to n).toArray.map(i => laguerre(i, alpha, x))),
41 | (x: Double) => DenseVector((0 to n).toArray.map(i => grad_laguerre(1)(i, alpha, x)))
42 | )
43 |
44 | }
--------------------------------------------------------------------------------
/helios/scripts/goes_flare_classification.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.PlasmaML.helios
2 | import ammonite.ops._
3 | import io.github.mandar2812.dynaml.repl.Router.main
4 | import org.joda.time._
5 | import org.platanios.tensorflow.api._
6 |
7 | @main
8 | def main(
9 | test_year: Int = 2003,
10 | re: Boolean = true,
11 | tmpdir: Path = root/"home"/System.getProperty("user.name")/"tmp",
12 | resFile: String = "mdi_xray_class_results.csv") = {
13 |
14 | //Data with MDI images
15 |
16 | val results = resFile
17 |
18 | print("Running experiment with test split from year: ")
19 | pprint.pprintln(test_year)
20 |
21 | val data = helios.generate_data_goes()
22 |
23 | println("Starting data set created.")
24 | println("Proceeding to load images & labels into Tensors ...")
25 | val flux_threshold = -5.5
26 |
27 | val test_start = new DateTime(test_year, 1, 1, 0, 0)
28 |
29 | val test_end = new DateTime(test_year, 12, 31, 23, 59)
30 |
31 | val tt_partition = (p: (DateTime, (Path, (Double, Double)))) =>
32 | if (p._1.isAfter(test_start) && p._1.isBefore(test_end) && p._2._2._2 >= flux_threshold) false
33 | else true
34 |
35 | val summary_dir = if(re) "mdi_wtloss_ext_resample_"+test_year else "mdi_wtloss_ext_"+test_year
36 |
37 | val res = helios.run_experiment_goes(
38 | data, tt_partition, resample = re,
39 | longWavelength = true)(
40 | summary_dir, 200000, tmpdir,
41 | arch = helios.learn.cnn_xray_class_v1,
42 | lossFunc = tf.learn.SparseSoftmaxCrossEntropy("Loss/CrossEntropy"))
43 |
44 |
45 | //Write the cross validation score in a results file
46 |
47 | val accuracy = res._3.results(0).sum()
48 |
49 | if(!exists(tmpdir/results)) write(tmpdir/results, "testyear,accuracy\n")
50 |
51 | write.append(tmpdir/results, s"$test_year,$accuracy\n")
52 |
53 | pprint.pprintln(res)
54 |
55 | res
56 | }
57 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/VariableAttribute.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.AttributeEntry;
4 |
5 | /**
6 | * Provides the description and per-variable entry values
7 | * for a CDF attribute with variable scope.
8 | *
9 | * @author Mark Taylor
10 | * @since 20 Jun 2013
11 | */
12 | public class VariableAttribute {
13 |
14 | private final String name_;
15 | private final AttributeEntry[] rEntries_;
16 | private final AttributeEntry[] zEntries_;
17 |
18 | /**
19 | * Constructor.
20 | *
21 | * @param name attribute name
22 | * @param rEntries rEntry values for this attribute
23 | * @param zEntries zEntry values for this attribute
24 | */
25 | public VariableAttribute( String name, AttributeEntry[] rEntries,
26 | AttributeEntry[] zEntries ) {
27 | name_ = name;
28 | rEntries_ = rEntries;
29 | zEntries_ = zEntries;
30 | }
31 |
32 | /**
33 | * Returns this attribute's name.
34 | *
35 | * @return attribute name
36 | */
37 | public String getName() {
38 | return name_;
39 | }
40 |
41 | /**
42 | * Returns the entry value that a given variable has for this attribute.
43 | * If the variable has no entry for this attribute, null is returned.
44 | *
45 | * @param variable CDF variable from the same CDF as this attribute
46 | * @return this attribute's value for variable
47 | */
48 | public AttributeEntry getEntry( Variable variable ) {
49 | AttributeEntry[] entries = variable.isZVariable() ? zEntries_
50 | : rEntries_;
51 | int ix = variable.getNum();
52 | return ix < entries.length ? entries[ ix ] : null;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/vanAllen/scripts/crresModels.scala:
--------------------------------------------------------------------------------
1 | import breeze.linalg.DenseVector
2 | import io.github.mandar2812.PlasmaML.vanAllen.{CRRESKernel, CRRESTest}
3 | import io.github.mandar2812.dynaml.kernels.{CompositeCovariance, DiracKernel, LocalSVMKernel}
4 | import io.github.mandar2812.dynaml.pipes.DataPipe
5 |
6 | // Test a GP model
7 | val waveletF = (x: Double) => math.cos(1.75*x)*math.exp(-1*x*x/2.0)
8 |
9 | CRRESTest(new RBFKernel(1.0), new DiracKernel(1.0), 1000, 1000, 3, 0.2)
10 |
11 | CRRESTest(
12 | new CauchyKernel(1.0),
13 | new DiracKernel(1.0), 1000, 1000,
14 | 3, 0.2)
15 |
16 | CRRESTest(
17 | new LaplacianKernel(1.0),
18 | new DiracKernel(1.0), 1000, 1000,
19 | 3, 0.2)
20 |
21 |
22 | val crresKern: CRRESKernel = new CRRESKernel(1.0, 0.9, 1.0, 1.0)
23 |
24 | //crresKern.blocked_hyper_parameters =
25 | // crresKern.rbfK.hyper_parameters
26 |
27 | //crresKern.rbfK.blocked_hyper_parameters = crresKern.rbfK.hyper_parameters
28 |
29 | val noiseKern:DiracKernel = new DiracKernel(0.4)
30 |
31 | noiseKern.blocked_hyper_parameters = noiseKern.hyper_parameters
32 |
33 | CRRESTest(
34 | crresKern,
35 | noiseKern, 1000, 1000,
36 | 2, 0.2)
37 |
38 |
39 | val waveletKern: LocalSVMKernel[DenseVector[Double]] = new WaveletKernel(waveletF)(1.1)
40 |
41 | val rbfKern: LocalSVMKernel[DenseVector[Double]] = new RBFKernel(1.5)
42 |
43 |
44 |
45 | val crresKern1: CompositeCovariance[DenseVector[Double]] = (waveletKern * rbfKern)
46 |
47 | crresKern1.blocked_hyper_parameters = waveletKern.hyper_parameters ++ rbfKern.hyper_parameters
48 |
49 | CRRESTest(waveletKern, new DiracKernel(0.84), 1000, 1000, 2, 0.2)
50 |
51 |
52 | // Test a feed forward neural network
53 | CRRESTest(
54 | 1, List("tansig", "linear"), List(10),
55 | 8000, 2000)
56 |
57 |
58 | CRRESTest(
59 | 0, List("logsig"), List(),
60 | 8000, 2000)
61 |
62 | CRRESTest(
63 | 0, List("linear"), List(),
64 | 6000, 1000)
65 |
66 |
67 | CRRESTest(1000)
--------------------------------------------------------------------------------
/helios/scripts/env.sc:
--------------------------------------------------------------------------------
1 | import ammonite.ops._
2 |
3 | val summary_dirs = Map(
4 | "tail-box" -> root / 'media / 'disk2 / 'scratch / System
5 | .getProperty("user.name") / 'tmp,
6 | "citronelle" -> root / 'home / System.getProperty("user.name") / 'tmp,
7 | "juniper" -> root / 'export / 'scratch3 / System
8 | .getProperty("user.name") / 'summaries,
9 | "wax" -> root / 'export / 'scratch2 / System
10 | .getProperty("user.name") / 'summaries
11 | )
12 |
13 | val data_dirs = Map(
14 | "citronelle" -> root / 'home / System
15 | .getProperty("user.name") / "data_repo" / 'helios,
16 | "juniper" -> root / 'export / 'scratch2 / System
17 | .getProperty("user.name") / "data_repo" / 'helios
18 | )
19 |
20 | val fte_data_dirs = Map(
21 | "citronelle" -> home / 'Downloads / 'fte,
22 | "juniper" -> home / 'Downloads / 'fte,
23 | "tail-box" -> root / 'media / 'disk2 / 'scratch / System
24 | .getProperty("user.name") / 'fte
25 | )
26 |
27 | val host: Option[String] = try {
28 | Some(
29 | java.net.InetAddress.getLocalHost().toString.split('/').head.split('.').head
30 | )
31 | } catch {
32 | case _: java.net.UnknownHostException => None
33 | case _: Exception => None
34 | }
35 |
36 | val default_summary_dir = home / 'tmp
37 | val default_fte_data_dir = home / 'Downloads / 'fte
38 | val default_data_dir = pwd / 'data
39 |
40 | val summary_dir = host match {
41 | case None => default_summary_dir
42 | case Some(host) =>
43 | if (summary_dirs.contains(host)) summary_dirs(host) else default_summary_dir
44 | }
45 |
46 | val data_dir = host match {
47 | case None => default_data_dir
48 | case Some(host) =>
49 | if (data_dirs.contains(host)) data_dirs(host) else default_data_dir
50 | }
51 |
52 | val fte_data_dir = host match {
53 | case None => default_fte_data_dir
54 | case Some(host) =>
55 | if (fte_data_dirs.contains(host)) fte_data_dirs(host)
56 | else default_fte_data_dir
57 | }
58 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/CdfInfo.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | /**
4 | * Encapsulates some global information about a CDF file.
5 | *
6 | * @author Mark Taylor
7 | * @since 20 Jun 2013
8 | */
9 | public class CdfInfo {
10 | private final boolean rowMajor_;
11 | private final int[] rDimSizes_;
12 | private final int leapSecondLastUpdated_;
13 |
14 | /**
15 | * Constructor.
16 | *
17 | * @param rowMajor true for row majority, false for column majority
18 | * @param rDimSizes array of dimension sizes for rVariables
19 | * @param leapSecondLastUpdated value of the GDR LeapSecondLastUpdated
20 | * field
21 | */
22 | public CdfInfo( boolean rowMajor, int[] rDimSizes,
23 | int leapSecondLastUpdated ) {
24 | rowMajor_ = rowMajor;
25 | rDimSizes_ = rDimSizes;
26 | leapSecondLastUpdated_ = leapSecondLastUpdated;
27 | }
28 |
29 | /**
30 | * Indicates majority of CDF arrays.
31 | *
32 | * @return true for row majority, false for column majority
33 | */
34 | public boolean getRowMajor() {
35 | return rowMajor_;
36 | }
37 |
38 | /**
39 | * Returns array dimensions for rVariables.
40 | *
41 | * @return array of dimension sizes for rVariables
42 | */
43 | public int[] getRDimSizes() {
44 | return rDimSizes_;
45 | }
46 |
47 | /**
48 | * Returns the date of the last leap second the CDF file knows about.
49 | * This is the value of the LeapSecondLastUpdated field from the GDR
50 | * (introduced at CDF v3.6). The value is an integer whose
51 | * decimal representation is of the form YYYYMMDD.
52 | * Values 0 and -1 have special meaning (no last leap second).
53 | *
54 | * @return last known leap second indicator
55 | */
56 | public int getLeapSecondLastUpdated() {
57 | return leapSecondLastUpdated_;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/omni/scripts/processOmniDstNARXRes.R:
--------------------------------------------------------------------------------
1 | #! /usr/bin/Rscript
2 | library(ggplot2)
3 | library(gridExtra)
4 | setwd("../../../DynaML/data/")
5 |
6 | df <- read.csv("OmniNARXRes.csv" ,
7 | header = FALSE, stringsAsFactors = FALSE,
8 | colClasses = rep("numeric",12),
9 | col.names = c("trainyear", "testyear",
10 | "order", "exogenousInputs","stepAhead",
11 | "modelSize", "numTest", "mae", "rmse",
12 | "rsq", "corr", "yi", "deltaT"))
13 |
14 |
15 | #colorless box plots
16 | p9 <- qplot(data = df, x = factor(modelSize), y = corr,
17 | facets = .~order, geom = "boxplot",
18 | xlab = "Model Size", ylab = "Cross Correlation")
19 |
20 | p10 <- qplot(data = df, x = factor(modelSize), y = yi,
21 | facets = .~order, geom = "boxplot",
22 | xlab = "Model Size", ylab = "Model Yield")
23 |
24 |
25 | p11 <- qplot(data = df, x = factor(modelSize), y = mae, geom = "boxplot",
26 | facets = .~order, xlab = "Model Size", ylab = "Mean Abs. Error")
27 |
28 | p12 <- qplot(data = df, x = factor(modelSize), y = deltaT, geom = "boxplot",
29 | facets = .~order, xlab = "Model Size", ylab = "Timing Error")
30 |
31 | grid.arrange(p11, p9, p10, p12, nrow = 2, ncol=2)
32 |
33 | #fill Model Size
34 |
35 | p17 <- qplot(data = df, x = factor(order), y = mae, fill = factor(modelSize),
36 | geom = "boxplot", xlab = "Model Size",
37 | ylab = "Mean Abs. Error")
38 |
39 | p18 <- qplot(data = df, x = factor(order), y = corr, fill = factor(modelSize),
40 | geom = "boxplot",
41 | xlab = "Model Size", ylab = "Cross Correlation")
42 |
43 | p19 <- qplot(data = df, x = factor(order), y = yi, fill = factor(modelSize),
44 | geom = "boxplot",
45 | xlab = "Model Size", ylab = "Model Yield")
46 |
47 | p20 <- qplot(data = df, x = factor(order), y = deltaT, fill = factor(modelSize),
48 | geom = "boxplot", xlab = "Model Size", ylab = "Timing Error")
49 |
50 | grid.arrange(p17, p18, p19, p20, nrow = 2, ncol=2)
51 |
--------------------------------------------------------------------------------
/helios/scripts/goes_extreme_events.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.PlasmaML.helios
2 | import ammonite.ops._
3 | import io.github.mandar2812.dynaml.repl.Router.main
4 | import org.joda.time._
5 |
6 | @main
7 | def main(
8 | test_year: Int = 2003,
9 | longWL: Boolean = false,
10 | re: Boolean = true,
11 | tmpdir: Path = root/"home"/System.getProperty("user.name")/"tmp",
12 | resFile: String = "mdi_ext_results.csv") = {
13 |
14 | //Data with MDI images
15 |
16 | val results = if(longWL) "longwl_"+resFile else resFile
17 |
18 | print("Running experiment with test split from year: ")
19 | pprint.pprintln(test_year)
20 |
21 | val data = helios.generate_data_goes()
22 |
23 | println("Starting data set created.")
24 | println("Proceeding to load images & labels into Tensors ...")
25 | val flux_threshold = if(longWL) -5.5d else -6.5d
26 |
27 | val test_start = new DateTime(test_year, 1, 1, 0, 0)
28 |
29 | val test_end = new DateTime(test_year, 12, 31, 23, 59)
30 |
31 | val tt_partition = if(longWL) {
32 |
33 | (p: (DateTime, (Path, (Double, Double)))) =>
34 | if (p._1.isAfter(test_start) && p._1.isBefore(test_end) && p._2._2._2 >= flux_threshold) false
35 | else true
36 |
37 | } else {
38 |
39 | (p: (DateTime, (Path, (Double, Double)))) =>
40 | if(p._1.isAfter(test_start) && p._1.isBefore(test_end) && p._2._2._1 >= flux_threshold) false
41 | else true
42 |
43 | }
44 |
45 | val summary_dir = if(re) "mdi_ext_resample_"+test_year else "mdi_ext_"+test_year
46 |
47 | val res = helios.run_experiment_goes(
48 | data, tt_partition, resample = re,
49 | longWavelength = longWL)(
50 | summary_dir, 200000, tmpdir,
51 | arch = helios.learn.cnn_goes_v1_1)
52 |
53 |
54 | //Write the cross validation score in a results file
55 |
56 | val accuracy = res._3.results(0).sum()
57 |
58 | if(!exists(tmpdir/results)) write(tmpdir/results, "testyear,accuracy\n")
59 |
60 | write.append(tmpdir/results, s"$test_year,$accuracy\n")
61 |
62 | pprint.pprintln(res)
63 |
64 | res
65 | }
--------------------------------------------------------------------------------
/helios/scripts/csss.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.{utils => dutils}
2 | import $exec.env
3 | import ammonite.ops.ImplicitWd._
4 | import _root_.io.github.mandar2812.dynaml.pipes._
5 |
6 | def experiments() =
7 | ls ! env.summary_dir |? (_.isDir) |? (_.segments.toSeq.last.contains("fte"))
8 |
9 |
10 | def scatter_plots_test(summary_dir: Path) =
11 | ls ! summary_dir |? (_.segments.toSeq.last.contains("scatter_test"))
12 |
13 | def scatter_plots_train(summary_dir: Path) =
14 | ls ! summary_dir |? (_.segments.toSeq.last.contains("scatter_train"))
15 |
16 | def test_preds(summary_dir: Path) =
17 | (ls ! summary_dir |? (_.segments.toSeq.last.contains("predictions_test")))
18 |
19 | def test_data(summary_dir: Path) =
20 | (ls ! summary_dir |? (_.segments.toSeq.last.contains("test_data")))
21 |
22 | def test_data_probs(summary_dir: Path) =
23 | (ls ! summary_dir |? (_.segments.toSeq.last.contains("probabilities_test")))
24 |
25 | def test_data_preds(summary_dir: Path) =
26 | (ls ! summary_dir |? (_.segments.toSeq.last.contains("predictions_test")))
27 |
28 |
29 | val script = pwd / 'helios / 'scripts / "visualise_tl.R"
30 |
31 | val script_ts_rec = pwd / 'helios / 'scripts / "visualise_ts_rec.R"
32 |
33 | val time_window = (48, 72)
34 |
35 | val avg_sw_6h = DataPipe(
36 | (xs: Seq[Double]) => xs.grouped(6).map(g => g.sum / g.length).toSeq
37 | )
38 |
39 | val max_sw_6h = DataPipe(
40 | (xs: Seq[Double]) => xs.grouped(6).map(g => g.max).toSeq
41 | )
42 |
43 | val median_sw_6h = DataPipe(
44 | (xs: Seq[Double]) => xs.grouped(6).map(g => dutils.median(g.toStream)).toSeq
45 | )
46 |
47 |
48 | val avg_sw_24h = DataPipe(
49 | (xs: Seq[Double]) => xs.grouped(24).map(g => g.sum / g.length).toSeq
50 | )
51 |
52 | val max_sw_24h = DataPipe(
53 | (xs: Seq[Double]) => xs.grouped(24).map(g => g.max).toSeq
54 | )
55 |
56 | val median_sw_24h = DataPipe(
57 | (xs: Seq[Double]) => xs.grouped(24).map(g => dutils.median(g.toStream)).toSeq
58 | )
59 |
60 | val fact = 3
61 | val base_iterations = 80000
62 | def ext_iterations = base_iterations * fact
63 |
64 | val base_it_pdt = 3
65 | def ext_it_pdt = base_it_pdt + 2
66 |
--------------------------------------------------------------------------------
/omni/scripts/MOGPAnalysis.R:
--------------------------------------------------------------------------------
1 | library(plyr)
2 | library(ggplot2)
3 | library(gridExtra)
4 | library(reshape2)
5 | library(latex2exp)
6 |
7 | palette1 <- c("#000000", "firebrick3", "forestgreen", "steelblue2")
8 | lines1 <- c("solid", "solid", "dotdash", "dotdash")
9 | setwd("~/Development/PlasmaML/data/")
10 |
11 |
12 | # Compare Brier score of naive models vs GP-ARX
13 | df <- read.csv("brier_scores.csv",
14 | header = FALSE,
15 | stringsAsFactors = FALSE,
16 | colClasses = rep("numeric", 2),
17 | col.names = c("prob","brier"))
18 |
19 |
20 | ggplot(df, aes(x=prob,y=brier)) +
21 | geom_path(size=1.25) + geom_hline(aes(yintercept=0.076, linetype = "dotdash"), show.legend = TRUE) +
22 | theme_gray(base_size = 22) +
23 | scale_colour_manual(values=palette1) +
24 | scale_linetype_manual(values = lines1, guide=FALSE) +
25 | xlab("Probability of Storm Jump") + ylab("Brier Score")
26 |
27 |
28 | arx_errorbars_pred <- read.csv("mogp_preds_2004_11_07-2.csv",
29 | header = FALSE,
30 | col.names = c("Dst", "predicted", "lower", "upper"))
31 | arx_errorbars_pred$time <- 1:nrow(arx_errorbars_pred)
32 |
33 | meltedPred <- melt(arx_errorbars_pred, id="time")
34 |
35 | ggplot(meltedPred, aes(x=time,y=value, colour=variable, linetype=variable)) +
36 | geom_line(size=1.35) +
37 | theme_gray(base_size = 22) +
38 | scale_colour_manual(values=palette1) +
39 | scale_linetype_manual(values = lines1, guide=FALSE) +
40 | xlab("Time (hours)") + ylab("Dst(t+3)")
41 |
42 | arx_onset_pred <- read.csv("mogp_onset_2004_11_07-2.csv",
43 | header = FALSE,
44 | col.names = c("p", "storm"))
45 | arx_onset_pred$time <- 1:nrow(arx_onset_pred)
46 |
47 | meltedPred1 <- melt(arx_onset_pred, id="time")
48 |
49 | ggplot(meltedPred1, aes(x=time,y=value, colour=variable, linetype=variable)) +
50 | geom_line(size=1.35) +
51 | theme_gray(base_size = 22) +
52 | scale_colour_manual(values=palette1) +
53 | scale_linetype_manual(values = lines1, guide=FALSE) +
54 | xlab("Time (hours)") + ylab("P(Dst(t+3) - Dst(t) < -70 nT)")
55 |
--------------------------------------------------------------------------------
/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/HeliosOmniTSMetrics.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.helios.core
2 |
3 | import io.github.mandar2812.dynaml.tensorflow.evaluation.MetricsTF
4 | import org.platanios.tensorflow.api._
5 | import _root_.io.github.mandar2812.dynaml.tensorflow._
6 |
7 | /**
8 | *
9 | * */
10 | class HeliosOmniTSMetrics[P: TF: IsFloatOrDouble](
11 | predictions: Tensor[P], targets: Tensor[P],
12 | size_causal_window: Int, time_scale: Tensor[P],
13 | p: Double = 2.0) extends
14 | MetricsTF[P](Seq("weighted_avg_err"), predictions, targets) {
15 |
16 | private[this] val scaling = Tensor(size_causal_window.toDouble-1d).castTo[P]
17 |
18 | override protected def run(): Tensor[P] = {
19 |
20 | val y = predictions(::, 0)
21 |
22 | val timelags = predictions(::, 1).sigmoid.multiply(scaling)
23 |
24 | val size_batch = predictions.shape(0)
25 |
26 | val repeated_times = dtf.stack(Seq.fill(size_causal_window)(timelags), axis = -1)
27 |
28 | val index_times = Tensor((0 until size_causal_window).map(_.toDouble)).reshape(Shape(size_causal_window)).castTo[P]
29 |
30 | val repeated_index_times = dtf.stack(Seq.fill(size_batch)(index_times), axis = 0).castTo[P]
31 |
32 | val repeated_preds = dtf.stack(Seq.fill(size_causal_window)(y), axis = -1)
33 |
34 | val repeated_time_scales = dtf.stack(Seq.fill(size_causal_window)(time_scale), axis = -1)
35 |
36 | val convolution_kernel = (repeated_index_times - repeated_times)
37 | .abs
38 | .pow(Tensor(p).castTo[P])
39 | .multiply(Tensor(-1.0/p).castTo[P])
40 | .divide(repeated_time_scales)
41 | .exp
42 |
43 | val weighted_loss_tensor =
44 | (repeated_preds - targets)
45 | .square
46 | .multiply(convolution_kernel)
47 | .sum(axes = 1)
48 | .divide(convolution_kernel.sum(axes = 1))
49 | .mean[Int]()
50 |
51 | weighted_loss_tensor
52 | }
53 |
54 | override def print(): Unit = {
55 | println("\nModel Performance: "+name)
56 | println("============================")
57 | println()
58 | println(names.head+": "+results.scalar.asInstanceOf[Double])
59 | println()
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/conf/DynaMLInit.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * External Imports
3 | * */
4 | //Ammonite imports
5 | import ammonite.ops._
6 | import ammonite.ops.ImplicitWd._
7 | //Import breeze for linear algebra
8 | import breeze.linalg.{DenseVector, DenseMatrix}
9 | import breeze.stats.distributions._
10 | //Apache Spark for big data support
11 | import org.apache.spark.SparkContext
12 | import org.apache.spark.SparkConf
13 | //Load Wisp-Highcharts for plotting
14 | import _root_.io.github.mandar2812.dynaml.graphics.charts.Highcharts._
15 | //Import spire implicits for definition of
16 | //fields, algebraic structures on primitive types
17 | import spire.implicits._
18 | /*
19 | * DynaML imports
20 | * */
21 | import io.github.mandar2812.dynaml.analysis.VectorField
22 | //The pipes API
23 | import io.github.mandar2812.dynaml.pipes._
24 | import io.github.mandar2812.dynaml.DynaMLPipe
25 | import io.github.mandar2812.dynaml.DynaMLPipe._
26 | //Load the DynaML model api members
27 | import io.github.mandar2812.dynaml.models._
28 | import io.github.mandar2812.dynaml.models.neuralnets._
29 | import io.github.mandar2812.dynaml.models.svm._
30 | import io.github.mandar2812.dynaml.models.lm._
31 | //Utility functions
32 | import io.github.mandar2812.dynaml.utils
33 | //Kernels for GP,SVM models
34 | import io.github.mandar2812.dynaml.kernels._
35 | //Load neural net primitives
36 | import io.github.mandar2812.dynaml.models.neuralnets.TransferFunctions._
37 | //The probability API
38 | import io.github.mandar2812.dynaml.probability._
39 | import io.github.mandar2812.dynaml.probability.distributions._
40 | //Wavelet API
41 | import io.github.mandar2812.dynaml.wavelets._
42 | //OpenML support
43 | //Renjin imports
44 | import javax.script._
45 | import org.renjin.script._
46 | import org.renjin.sexp._
47 |
48 | import io.github.mandar2812.PlasmaML.omni._
49 | import io.github.mandar2812.PlasmaML.vanAllen._
50 | import io.github.mandar2812.PlasmaML.dynamics.diffusion._
51 |
52 | val r_engine_factory = new RenjinScriptEngineFactory()
53 | implicit val renjin = r_engine_factory.getScriptEngine()
54 | val r: String => SEXP = (s: String) => renjin.eval(s).asInstanceOf[SEXP]
55 | val R: java.io.File => SEXP = (f: java.io.File) => renjin.eval(f).asInstanceOf[SEXP]
56 |
57 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/AttributeDescriptorRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type Attribute Descriptor Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class AttributeDescriptorRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long adrNext;
20 | @CdfField @OffsetField public final long agrEdrHead;
21 | @CdfField public final int scope;
22 | @CdfField public final int num;
23 | @CdfField public final int nGrEntries;
24 | @CdfField public final int maxGrEntry;
25 | @CdfField public final int rfuA;
26 | @CdfField @OffsetField public final long azEdrHead;
27 | @CdfField public final int nZEntries;
28 | @CdfField public final int maxZEntry;
29 | @CdfField public final int rfuE;
30 | @CdfField public final String name;
31 |
32 | /**
33 | * Constructor.
34 | *
35 | * @param plan basic record info
36 | * @param nameLeng number of characters used for attribute names
37 | */
38 | public AttributeDescriptorRecord( RecordPlan plan, int nameLeng )
39 | throws IOException {
40 | super( plan, "ADR", 4 );
41 | Buf buf = plan.getBuf();
42 | Pointer ptr = plan.createContentPointer();
43 | this.adrNext = buf.readOffset( ptr );
44 | this.agrEdrHead = buf.readOffset( ptr );
45 | this.scope = buf.readInt( ptr );
46 | this.num = buf.readInt( ptr );
47 | this.nGrEntries = buf.readInt( ptr );
48 | this.maxGrEntry = buf.readInt( ptr );
49 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
50 | this.azEdrHead = buf.readOffset( ptr );
51 | this.nZEntries = buf.readInt( ptr );
52 | this.maxZEntry = buf.readInt( ptr );
53 | this.rfuE = checkIntValue( buf.readInt( ptr ), -1 );
54 | this.name = buf.readAsciiString( ptr, nameLeng );
55 | checkEndRecord( ptr );
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/data/CDFLeapSeconds.txt:
--------------------------------------------------------------------------------
1 | ; Source:
2 | ; Updated: 20150201
3 | ; Leap Seconds Table - used by CDF
4 | ; Update it when a leap second(s) is added.
5 | ; Comment lines starts with ";" at column 1.
6 | ; Year Month Day Leap Seconds Drift
7 | 1960 1 1 1.4178180 37300.0 0.001296
8 | 1961 1 1 1.4228180 37300.0 0.001296
9 | 1961 8 1 1.3728180 37300.0 0.001296
10 | 1962 1 1 1.8458580 37665.0 0.0011232
11 | 1963 11 1 1.9458580 37665.0 0.0011232
12 | 1964 1 1 3.2401300 38761.0 0.001296
13 | 1964 4 1 3.3401300 38761.0 0.001296
14 | 1964 9 1 3.4401300 38761.0 0.001296
15 | 1965 1 1 3.5401300 38761.0 0.001296
16 | 1965 3 1 3.6401300 38761.0 0.001296
17 | 1965 7 1 3.7401300 38761.0 0.001296
18 | 1965 9 1 3.8401300 38761.0 0.001296
19 | 1966 1 1 4.3131700 39126.0 0.002592
20 | 1968 2 1 4.2131700 39126.0 0.002592
21 | 1972 1 1 10.0 0.0 0.0
22 | 1972 7 1 11.0 0.0 0.0
23 | 1973 1 1 12.0 0.0 0.0
24 | 1974 1 1 13.0 0.0 0.0
25 | 1975 1 1 14.0 0.0 0.0
26 | 1976 1 1 15.0 0.0 0.0
27 | 1977 1 1 16.0 0.0 0.0
28 | 1978 1 1 17.0 0.0 0.0
29 | 1979 1 1 18.0 0.0 0.0
30 | 1980 1 1 19.0 0.0 0.0
31 | 1981 7 1 20.0 0.0 0.0
32 | 1982 7 1 21.0 0.0 0.0
33 | 1983 7 1 22.0 0.0 0.0
34 | 1985 7 1 23.0 0.0 0.0
35 | 1988 1 1 24.0 0.0 0.0
36 | 1990 1 1 25.0 0.0 0.0
37 | 1991 1 1 26.0 0.0 0.0
38 | 1992 7 1 27.0 0.0 0.0
39 | 1993 7 1 28.0 0.0 0.0
40 | 1994 7 1 29.0 0.0 0.0
41 | 1996 1 1 30.0 0.0 0.0
42 | 1997 7 1 31.0 0.0 0.0
43 | 1999 1 1 32.0 0.0 0.0
44 | 2006 1 1 33.0 0.0 0.0
45 | 2009 1 1 34.0 0.0 0.0
46 | 2012 7 1 35.0 0.0 0.0
47 | 2015 7 1 36.0 0.0 0.0
48 |
--------------------------------------------------------------------------------
/helios/scripts/stability_analysis.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.tensorflow.data._
2 | import _root_.io.github.mandar2812.dynaml.tensorflow._
3 | import _root_.io.github.mandar2812.PlasmaML.helios.fte
4 | import _root_.io.github.mandar2812.PlasmaML.helios
5 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
6 |
7 | import _root_.org.platanios.tensorflow.api._
8 |
9 | import $exec.helios.scripts.csss
10 | import $exec.helios.scripts.env
11 |
12 | val exp_dirs =
13 | (1 to 4).flatMap(
14 | i =>
15 | ls ! home / "tmp" / s"results_exp$i" |? (
16 | p => p.isDir && !p.segments.last.contains("bs_")
17 | )
18 | )
19 |
20 | val extract_state = (p: Path) => {
21 | val lines = read.lines ! p / "state.csv"
22 | lines.head.split(",").zip(lines.last.split(",").map(_.toDouble)).toMap
23 | }
24 |
25 | val params_enc = Encoder(
26 | identityPipe[Map[String, Double]],
27 | identityPipe[Map[String, Double]]
28 | )
29 |
30 | val stabilities = exp_dirs.map(exp_dir => {
31 |
32 | val state = extract_state(exp_dir)
33 |
34 | val triple = timelag.utils.read_cdt_model_preds(
35 | exp_dir / "test_predictions.csv",
36 | exp_dir / "test_probabilities.csv",
37 | exp_dir / "test_data_targets.csv"
38 | )
39 |
40 | timelag.utils.compute_stability_metrics(
41 | triple._1,
42 | triple._2,
43 | triple._3,
44 | state,
45 | params_enc
46 | )
47 |
48 | })
49 |
50 | val fte_exp = csss.experiments()
51 |
52 | val fte_stability = fte_exp.map(exp_dir => {
53 |
54 | val state = extract_state(exp_dir)
55 |
56 | val preds =
57 | (ls ! exp_dir |? (_.segments.last.contains("predictions_test")))(0)
58 | val probs =
59 | (ls ! exp_dir |? (_.segments.last.contains("probabilities_test")))(0)
60 |
61 | require(
62 | preds.segments.last.split('.').head.split('_').last == probs.segments.last
63 | .split('.')
64 | .head
65 | .split('_')
66 | .last
67 | )
68 |
69 | val fte_data = (ls ! exp_dir |? (_.segments.last.contains("test_data"))).last
70 |
71 | val triple = fte.data.fte_model_preds(preds, probs, fte_data)
72 |
73 | timelag.utils.compute_stability_metrics(
74 | triple._1,
75 | triple._2,
76 | triple._3,
77 | state,
78 | params_enc
79 | )
80 | })
81 |
--------------------------------------------------------------------------------
/helios/scripts/dump_fte_omni_data.sc:
--------------------------------------------------------------------------------
1 | import _root_.ammonite.ops._
2 | import _root_.org.joda.time._
3 | import _root_.org.joda.time.format.DateTimeFormat
4 | import _root_.breeze.linalg._
5 | import _root_.io.github.mandar2812.dynaml.tensorflow._
6 | import _root_.io.github.mandar2812.dynaml.tensorflow.data._
7 | import _root_.io.github.mandar2812.PlasmaML.helios.fte
8 | import _root_.io.github.mandar2812.PlasmaML.omni.{OMNIData, OMNILoader}
9 |
10 | DateTimeZone.setDefault(DateTimeZone.UTC)
11 |
12 | def solar_wind_time_series(
13 | start: DateTime,
14 | end: DateTime
15 | ): ZipDataSet[DateTime, Seq[Double]] = {
16 |
17 | val omni_data_path = pwd / 'data
18 |
19 | val load_omni_file =
20 | fileToStream >
21 | replaceWhiteSpaces >
22 | extractTrainingFeatures(
23 | OMNIData.dateColumns ++ List(OMNIData.Quantities.V_SW),
24 | OMNIData.columnFillValues
25 | ) >
26 | OMNILoader.processWithDateTime
27 |
28 | dtfdata
29 | .dataset(start.getYear to end.getYear)
30 | .map(
31 | DataPipe(
32 | (i: Int) => omni_data_path.toString() + "/" + OMNIData.getFilePattern(i)
33 | )
34 | )
35 | .flatMap(load_omni_file)
36 | .to_zip(identityPipe[(DateTime, Seq[Double])])
37 |
38 | }
39 |
40 | val deltaTFTE = 0
41 | val fteStep = 0
42 | val fte_data_path = home / 'Downloads / 'fte
43 | val log_scale_fte = false
44 | val latitude_limit = 90d
45 | val conv_flag = false
46 | val log_scale_omni = false
47 |
48 | val start = new DateTime(2007, 1, 1, 0, 0, 0)
49 | val end = new DateTime(2018, 12, 31, 23, 59, 59)
50 |
51 | val fte_data = fte.data.load_fte_data_bdv(
52 | fte_data_path,
53 | fte.data.carrington_rotations,
54 | log_scale_fte,
55 | start,
56 | end
57 | )(deltaTFTE, fteStep, latitude_limit, conv_flag)
58 |
59 | val omni_data =
60 | solar_wind_time_series(start, end)
61 |
62 | val data = fte_data.join(omni_data)
63 |
64 | type PAT = (DateTime, (DenseVector[Double], Seq[Double]))
65 |
66 | val dump_file = home / 'Downloads / s"fte_sw_${start.getYear}-${end.getYear}.csv"
67 |
68 | data.foreach(DataPipe((p: PAT) => {
69 |
70 | val line = Seq(p._1.toString("YYYY-MM-dd-HH"), p._2._1.toArray.mkString(","), p._2._2.mkString(",")).mkString(",")+"\n"
71 |
72 | write.append(
73 | dump_file,
74 | line
75 | )
76 | }))
77 |
--------------------------------------------------------------------------------
/omni/src/main/scala/io/github/mandar2812/PlasmaML/omni/OmniBurton.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.omni
2 |
3 | import io.github.mandar2812.PlasmaML.omni.OmniOSA._
4 | import io.github.mandar2812.dynaml.DynaMLPipe.extractTimeSeries
5 | import io.github.mandar2812.dynaml.pipes.{DataPipe, IterableDataPipe}
6 |
7 | /**
8 | * Contains work-flows for running experiments
9 | * on hybrid GP-Burton injection inference and
10 | * prediction models.
11 | *
12 | * The basic premise of this code is the Burton ODE
13 | * for time evolution of the Dst index
14 | *
15 | * d Dst(t)/dt + lambda*Dst(t) = Q(t)
16 | *
17 | * Q(t) ~ GP(0, K(t, t'))
18 | *
19 | * @author mandar2812 30/03/2017.
20 | * */
21 | object OmniBurton {
22 |
23 | /**
24 | * @return A [[DataPipe]] that processes date segments [[DateSection]] and
25 | * returns a stream of (t, Y(t)) where Y(t) is the signal/quantity chosen in
26 | * [[OmniOSA.targetColumn]]
27 | * */
28 | def processTimeSegment = DataPipe((dateLimits: DateSection) => {
29 | //For the given date limits generate the relevant data
30 | //The file name to read from
31 | val fileName = dataDir+"omni2_"+dateLimits._1.split("/").head+".csv"
32 |
33 | val (startStamp, endStamp) = (
34 | formatter.parseDateTime(dateLimits._1).minusHours(0).getMillis/1000.0,
35 | formatter.parseDateTime(dateLimits._2).getMillis/1000.0)
36 |
37 | //Set up a processing pipeline for the file
38 | val processFile = OMNILoader.omniFileToStream(40, List()) >
39 | extractTimeSeries((year,day,hour) => {
40 | dayofYearformatter.parseDateTime(
41 | year.toInt.toString + "/" + day.toInt.toString +
42 | "/" + hour.toInt.toString).getMillis/1000.0 }) >
43 | IterableDataPipe((couple: (Double, Double)) =>
44 | couple._1 >= startStamp && couple._1 <= endStamp)
45 |
46 | //Apply the pipeline to the file
47 | processFile(fileName)
48 | })
49 |
50 | val prepareData = DataPipe((s: Stream[(Double, Double)]) => {
51 | val tmin = s.map(_._1).min
52 | //subtract tmin from time index of each data point
53 | s.map(c => (c._1-tmin, c._2))
54 | })
55 |
56 | val getIncrements = DataPipe((s: Stream[(Double, Double)]) => {
57 | s.sliding(2).map(ss => (ss.last._1, ss.last._2 - ss.head._2))
58 | })
59 |
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/GlobalDescriptorRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type Global Descriptor Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class GlobalDescriptorRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long rVdrHead;
20 | @CdfField @OffsetField public final long zVdrHead;
21 | @CdfField @OffsetField public final long adrHead;
22 | @CdfField public final long eof;
23 | @CdfField public final int nrVars;
24 | @CdfField public final int numAttr;
25 | @CdfField public final int rMaxRec;
26 | @CdfField public final int rNumDims;
27 | @CdfField public final int nzVars;
28 | @CdfField @OffsetField public final long uirHead;
29 | @CdfField public final int rfuC;
30 | @CdfField public final int leapSecondLastUpdated;
31 | @CdfField public final int rfuE;
32 | @CdfField public final int[] rDimSizes;
33 |
34 | /**
35 | * Constructor.
36 | *
37 | * @param plan basic record information
38 | */
39 | public GlobalDescriptorRecord( RecordPlan plan ) throws IOException {
40 | super( plan, "GDR", 2 );
41 | Buf buf = plan.getBuf();
42 | Pointer ptr = plan.createContentPointer();
43 | this.rVdrHead = buf.readOffset( ptr );
44 | this.zVdrHead = buf.readOffset( ptr );
45 | this.adrHead = buf.readOffset( ptr );
46 | this.eof = buf.readOffset( ptr );
47 | this.nrVars = buf.readInt( ptr );
48 | this.numAttr = buf.readInt( ptr );
49 | this.rMaxRec = buf.readInt( ptr );
50 | this.rNumDims = buf.readInt( ptr );
51 | this.nzVars = buf.readInt( ptr );
52 | this.uirHead = buf.readOffset( ptr );
53 | this.rfuC = checkIntValue( buf.readInt( ptr ), 0 );
54 | this.leapSecondLastUpdated = buf.readInt( ptr );
55 | this.rfuE = checkIntValue( buf.readInt( ptr ), -1 );
56 | this.rDimSizes = readIntArray( buf , ptr, this.rNumDims );
57 | checkEndRecord( ptr );
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/omni/scripts/gpOmniOrder.R:
--------------------------------------------------------------------------------
1 | library(plyr)
2 | library(ggplot2)
3 | library(gridExtra)
4 | library(reshape2)
5 | library(latex2exp)
6 |
7 |
8 | df <- read.csv("Nov_4_2016_alt_7_1_OmniARXStormsRes.csv",
9 | header = FALSE, stringsAsFactors = TRUE,
10 | col.names = c("eventID","stormCat","order", "modelSize",
11 | "rmse", "corr", "deltaDstMin", "DstMin",
12 | "deltaT"))
13 |
14 |
15 | df1 <- read.csv("Nov_4_2016_alt_6_1_OmniARXStormsRes.csv",
16 | header = FALSE, stringsAsFactors = TRUE,
17 | col.names = c("eventID","stormCat","order", "modelSize",
18 | "rmse", "corr", "deltaDstMin", "DstMin",
19 | "deltaT"))
20 |
21 | df2 <- read.csv("Nov_4_2016_alt_5_1_OmniARXStormsRes.csv",
22 | header = FALSE, stringsAsFactors = TRUE,
23 | col.names = c("eventID","stormCat","order", "modelSize",
24 | "rmse", "corr", "deltaDstMin", "DstMin",
25 | "deltaT"))
26 |
27 | df3 <- read.csv("Nov_4_2016_alt_4_1_OmniARXStormsRes.csv",
28 | header = FALSE, stringsAsFactors = TRUE,
29 | col.names = c("eventID","stormCat","order", "modelSize",
30 | "rmse", "corr", "deltaDstMin", "DstMin",
31 | "deltaT"))
32 |
33 | df4 <- read.csv("Nov_4_2016_alt_3_1_OmniARXStormsRes.csv",
34 | header = FALSE, stringsAsFactors = TRUE,
35 | col.names = c("eventID","stormCat","order", "modelSize",
36 | "rmse", "corr", "deltaDstMin", "DstMin",
37 | "deltaT"))
38 |
39 | df5 <- read.csv("Nov_4_2016_alt_2_1_OmniARXStormsRes.csv",
40 | header = FALSE, stringsAsFactors = TRUE,
41 | col.names = c("eventID","stormCat","order", "modelSize",
42 | "rmse", "corr", "deltaDstMin", "DstMin",
43 | "deltaT"))
44 |
45 | dfFinal <- rbind(df, df1, df2, df3, df4)
46 |
47 | qplot(data = dfFinal, x = factor(order), y = rmse,
48 | geom = "boxplot",
49 | xlab = "Model Order", ylab = "Model RMSE") +
50 | theme_gray(base_size = 22)
51 |
52 | ggsave(filename = "Model_RMSE_validationStorms.png", scale = 2.0)
--------------------------------------------------------------------------------
/helios/scripts/causal_time_lag_tuning.sc:
--------------------------------------------------------------------------------
1 | import $exec.helios.scripts.{
2 | tl_const_v_tuning => tuning_exp2,
3 | tl_const_a_tuning => tuning_exp3,
4 | tl_softplus_tuning => tuning_exp4}
5 |
6 |
7 | val res_tune_exp2 = tuning_exp2.main(
8 | d = 10, size_training = 8000, size_test = 2000,
9 | sliding_window = 20,
10 | noise = 0.7, noiserot = 0.001,
11 | alpha = 0.02, train_test_separate = true,
12 | num_neurons = Seq(30, 25),
13 | iterations = 150000, iterations_tuning = 10000,
14 | miniBatch = 1024, optimizer = tf.train.AdaDelta(0.01),
15 | prior_type = Seq(helios.learn.cdt_loss.KullbackLeibler),
16 | target_prob = Seq(helios.learn.cdt_loss.Uniform, helios.learn.cdt_loss.Boltzmann),
17 | dist_type = "default", num_samples = 20,
18 | hyper_optimizer = "gs",
19 | hyp_opt_iterations = Some(8),
20 | regularization_type = "L2")
21 |
22 |
23 | timelag.organize_results(res_tune_exp2, home/"tmp"/"results_exp2")
24 |
25 | val res_tune_exp3 = tuning_exp3.main(
26 | d = 10, size_training = 8000, size_test = 2000,
27 | sliding_window = 20,
28 | noise = 0.7, noiserot = 0.001,
29 | alpha = 0.02, train_test_separate = true,
30 | num_neurons = Seq(30, 25),
31 | iterations = 150000, iterations_tuning = 10000,
32 | miniBatch = 1024, optimizer = tf.train.AdaDelta(0.01),
33 | prior_type = Seq(helios.learn.cdt_loss.KullbackLeibler),
34 | target_prob = Seq(helios.learn.cdt_loss.Uniform, helios.learn.cdt_loss.Boltzmann),
35 | dist_type = "default", num_samples = 20,
36 | hyper_optimizer = "gs",
37 | hyp_opt_iterations = Some(8),
38 | regularization_type = "L2")
39 |
40 |
41 | timelag.organize_results(res_tune_exp3, home/"tmp"/"results_exp3")
42 |
43 |
44 | val res_tune_exp4 = tuning_exp4.main(
45 | d = 10,
46 | size_training = 8000, size_test = 2000,
47 | sliding_window = 20,
48 | noise = 0.7, noiserot = 0.001,
49 | alpha = 0.02, train_test_separate = true,
50 | num_neurons = Seq(30, 25),
51 | iterations = 150000, iterations_tuning = 10000,
52 | miniBatch = 1024, optimizer = tf.train.AdaDelta(0.01),
53 | prior_type = Seq(helios.learn.cdt_loss.KullbackLeibler),
54 | target_prob = Seq(helios.learn.cdt_loss.Uniform, helios.learn.cdt_loss.Boltzmann),
55 | dist_type = "default", num_samples = 20,
56 | hyper_optimizer = "gs",
57 | hyp_opt_iterations = Some(8),
58 | regularization_type = "L2")
59 |
60 |
61 | timelag.organize_results(res_tune_exp4, home/"tmp"/"results_exp4")
62 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/DataReader.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import java.io.IOException;
4 | import java.lang.reflect.Array;
5 |
6 | import io.github.mandar2812.PlasmaML.cdf.Buf;
7 | import io.github.mandar2812.PlasmaML.cdf.DataType;
8 |
9 | /**
10 | * Reads items with a given data type from a buffer into an array.
11 | *
12 | * @author Mark Taylor
13 | * @since 20 Jun 2013
14 | */
15 | public class DataReader {
16 |
17 | private final DataType dataType_;
18 | private final int nelPerItem_;
19 | private final int nItem_;
20 |
21 | /**
22 | * Constructor.
23 | *
24 | * @param dataType data type
25 | * @param nelPerItem number of dataType elements per read item;
26 | * usually 1 except for character data
27 | * @param nItem number of items of given data type in the array,
28 | * for scalar records it will be 1
29 | */
30 | public DataReader( DataType dataType, int nelPerItem, int nItem ) {
31 | dataType_ = dataType;
32 | nelPerItem_ = nelPerItem;
33 | nItem_ = nItem;
34 | }
35 |
36 | /**
37 | * Creates a workspace array which can contain a value read for one record.
38 | * The return value will be an array of a primitive type or String.
39 | *
40 | * @return workspace array for this reader
41 | */
42 | public Object createValueArray() {
43 | return Array.newInstance( dataType_.getArrayElementClass(),
44 | nItem_ * dataType_.getGroupSize() );
45 | }
46 |
47 | /**
48 | * Reads a value from a data buffer into a workspace array.
49 | *
50 | * @param buf data buffer
51 | * @param offset byte offset into buf of data start
52 | * @param valueArray object created by createValueArray
53 | * into which results will be read
54 | */
55 | public void readValue(Buf buf, long offset, Object valueArray )
56 | throws IOException {
57 | dataType_.readValues( buf, offset, nelPerItem_, valueArray, nItem_ );
58 | }
59 |
60 | /**
61 | * Returns the size in bytes of one record as stored in the data buffer.
62 | *
63 | * @return record size in bytes
64 | */
65 | public int getRecordSize() {
66 | return dataType_.getByteCount() * nelPerItem_ * nItem_;
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/implicits.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.dynaml.DynaMLPipe
5 | import io.github.mandar2812.dynaml.pipes.Encoder
6 |
7 | /**
8 | * Defines implicit transformations commonly used in
9 | * radial diffusion analysis.
10 | *
11 | * @author mandar2812 date 27/06/2017.
12 | * */
13 | object implicits {
14 |
15 | implicit val praramsEncTuple4: Encoder[(Double, Double, Double, Double), (Double, Double, Double, Double)] =
16 | Encoder(
17 | DynaMLPipe.identityPipe[(Double, Double, Double, Double)],
18 | DynaMLPipe.identityPipe[(Double, Double, Double, Double)])
19 |
20 | implicit val paramsEncBDV: Encoder[DenseVector[Double], (Double, Double, Double, Double)] = Encoder(
21 | (x: DenseVector[Double]) => (x(0), x(1), x(2), x(3)),
22 | (x: (Double, Double, Double, Double)) => DenseVector(x._1, x._2, x._3, x._4)
23 | )
24 |
25 | implicit val paramsEncSeq: Encoder[Seq[Double], (Double, Double, Double, Double)] = Encoder(
26 | (x: Seq[Double]) => (x.head, x(1), x(2), x(3)),
27 | (x: (Double, Double, Double, Double)) => Seq(x._1, x._2, x._3, x._4)
28 | )
29 |
30 | implicit val paramsEncList: Encoder[List[Double], (Double, Double, Double, Double)] = Encoder(
31 | (x: List[Double]) => (x.head, x(1), x(2), x(3)),
32 | (x: (Double, Double, Double, Double)) => List(x._1, x._2, x._3, x._4)
33 | )
34 |
35 | implicit val paramsEncArr: Encoder[Array[Double], (Double, Double, Double, Double)] = Encoder(
36 | (x: Array[Double]) => (x.head, x(1), x(2), x(3)),
37 | (x: (Double, Double, Double, Double)) => Array(x._1, x._2, x._3, x._4)
38 | )
39 |
40 | implicit val paramsEncMap: Encoder[Map[String, Double], (Double, Double, Double, Double)] = Encoder(
41 | (x: Map[String, Double]) => (x("alpha"), x("beta"), x("a"), x("b")),
42 | (x: (Double, Double, Double, Double)) => Map(
43 | "alpha" -> x._1, "beta" -> x._2,
44 | "a" -> x._3, "b" -> x._4)
45 | )
46 |
47 | def paramsEncMap(prefix: String): Encoder[Map[String, Double], (Double, Double, Double, Double)] = Encoder(
48 | (x: Map[String, Double]) => (x(prefix+"/alpha"), x(prefix+"/beta"), x(prefix+"/a"), x(prefix+"/b")),
49 | (x: (Double, Double, Double, Double)) => Map(
50 | prefix+"/alpha" -> x._1, prefix+"/beta" -> x._2,
51 | prefix+"/a" -> x._3, prefix+"/b" -> x._4)
52 | )
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/RunLengthInputStream.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | import java.io.IOException;
4 | import java.io.InputStream;
5 |
6 | /**
7 | * Decompression stream for CDF's version of Run Length Encoding.
8 | *
9 | * The compressed stream is just like the uncompressed one,
10 | * except that a byte with the special value V is followed by
11 | * a byte giving the number of additional bytes V to consider present
12 | * in the stream.
13 | * Thus the compressed stream:
14 | *
15 | * 1 2 3 0 0 4 5 6 0 2
16 | *
17 | * is decompressed as
18 | *
19 | * 1 2 3 0 4 5 6 0 0 0
20 | *
21 | * (assuming a special value V=0).
22 | *
23 | * This format was deduced from reading the cdfrle.c source file
24 | * from the CDF distribution.
25 | *
26 | * @author Mark Taylor
27 | * @since 17 May 2013
28 | */
29 | public class RunLengthInputStream extends InputStream {
30 |
31 | private final InputStream base_;
32 | private final int rleVal_;
33 | private int vCount_;
34 |
35 | /**
36 | * Constructor.
37 | *
38 | * @param base input stream containing RLE-compressed data
39 | * @param rleVal the byte value whose run lengths are compressed
40 | * (always zero for CDF as far as I can tell)
41 | */
42 | public RunLengthInputStream( InputStream base, byte rleVal ) {
43 | base_ = base;
44 | rleVal_ = rleVal & 0xff;
45 | }
46 |
47 | @Override
48 | public int read() throws IOException {
49 | if ( vCount_ > 0 ) {
50 | vCount_--;
51 | return rleVal_;
52 | }
53 | else {
54 | int b = base_.read();
55 | if ( b == rleVal_ ) {
56 | int c = base_.read();
57 | if ( c >= 0 ) {
58 | vCount_ = c;
59 | return rleVal_;
60 | }
61 | else {
62 | throw new CdfFormatException( "Bad RLE data" );
63 | }
64 | }
65 | else {
66 | return b;
67 | }
68 | }
69 | }
70 |
71 | @Override
72 | public int available() throws IOException {
73 | return base_.available() + vCount_;
74 | }
75 |
76 | @Override
77 | public void close() throws IOException {
78 | base_.close();
79 | }
80 |
81 | @Override
82 | public boolean markSupported() {
83 | return false;
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/helios/scripts/csss_job.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.{utils => dutils}
2 | import $file.csss
3 | import $file.csss_pdt_model_tuning
4 | import $file.csss_so_tuning
5 | import $file.env
6 | import _root_.io.github.mandar2812.dynaml.repl.Router.main
7 | import org.joda.time._
8 | import ammonite.ops._
9 | import ammonite.ops.ImplicitWd._
10 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
11 | import _root_.io.github.mandar2812.PlasmaML.omni.OMNIData
12 | import org.platanios.tensorflow.api._
13 |
14 | @main
15 | def main(
16 | csss_job_id: String = "",
17 | test_year: Int = 2015,
18 | test_month: Int = 10,
19 | test_rotation: Int = -1,
20 | network_size: Seq[Int] = Seq(50, 50)
21 | ) = {
22 |
23 | val dt = DateTime.now()
24 |
25 | val csss_exp = csss_pdt_model_tuning(
26 | csss_job_id = if(csss_job_id != "") Some(csss_job_id) else None,
27 | start_year = 2008,
28 | end_year = 2016,
29 | test_year = test_year,
30 | test_month = test_month,
31 | test_rotation = if(test_rotation == -1) None else Some(test_rotation),
32 | crop_latitude = 90d,
33 | sw_threshold = 600d,
34 | fraction_pca = 1d,
35 | fte_step = 0,
36 | history_fte = 0,
37 | log_scale_omni = false,
38 | log_scale_fte = true,
39 | time_window = csss.time_window,
40 | ts_transform_output = csss.median_sw_6h,
41 | network_size = network_size,
42 | use_persistence = true,
43 | activation_func = (i: Int) => timelag.utils.getReLUAct3[Double](1, 1, i, 0f),
44 | hyper_optimizer = "gs",
45 | num_samples = 4,
46 | quantity = OMNIData.Quantities.V_SW,
47 | omni_ext = Seq(
48 | OMNIData.Quantities.sunspot_number,
49 | OMNIData.Quantities.F10_7
50 | ),
51 | reg_type = "L2",
52 | batch_size = 256,
53 | max_iterations = csss.ext_iterations,
54 | max_iterations_tuning = csss.base_iterations,
55 | pdt_iterations_tuning = csss.base_it_pdt,
56 | pdt_iterations_test = csss.ext_it_pdt,
57 | optimization_algo = tf.train.Adam(0.001f),
58 | summary_dir = env.summary_dir,
59 | get_training_preds = true,
60 | data_scaling = "hybrid",
61 | use_copula = true,
62 | existing_exp = None
63 | )
64 |
65 | try {
66 | %%(
67 | 'Rscript,
68 | csss.script,
69 | csss_exp.results.summary_dir,
70 | csss.scatter_plots_test(csss_exp.results.summary_dir).last,
71 | "test_"
72 | )
73 | } catch {
74 | case e: Exception => e.printStackTrace()
75 | }
76 |
77 |
78 | println("CDT Model Performance:")
79 | csss_exp.results.metrics_test.get.print()
80 |
81 | }
82 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/CdfDescriptorRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Field data for CDF record of type CDF Descriptor Record.
11 | *
12 | * @author Mark Taylor
13 | * @since 19 Jun 2013
14 | */
15 | public class CdfDescriptorRecord extends Record {
16 |
17 | @CdfField
18 | @OffsetField
19 | public final long gdrOffset;
20 | @CdfField public final int version;
21 | @CdfField public final int release;
22 | @CdfField public final int encoding;
23 | @CdfField public final int flags;
24 | @CdfField public final int rfuA;
25 | @CdfField public final int rfuB;
26 | @CdfField public final int increment;
27 | @CdfField public final int rfuD;
28 | @CdfField public final int rfuE;
29 | public final String[] copyright;
30 |
31 | /**
32 | * Constructor.
33 | *
34 | * @param plan basic record information
35 | */
36 | public CdfDescriptorRecord( RecordPlan plan ) throws IOException {
37 | super( plan, "CDR", 1 );
38 | Buf buf = plan.getBuf();
39 | Pointer ptr = plan.createContentPointer();
40 | this.gdrOffset = buf.readOffset( ptr );
41 | this.version = buf.readInt( ptr );
42 | this.release = buf.readInt( ptr );
43 | this.encoding = buf.readInt( ptr );
44 | this.flags = buf.readInt( ptr );
45 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
46 | this.rfuB = checkIntValue( buf.readInt( ptr ), 0 );
47 | this.increment = buf.readInt( ptr );
48 | this.rfuD = checkIntValue( buf.readInt( ptr ), -1 );
49 | this.rfuE = checkIntValue( buf.readInt( ptr ), -1 );
50 | int crLeng = versionAtLeast( 2, 5 ) ? 256 : 1945;
51 | this.copyright = toLines( buf.readAsciiString( ptr, crLeng ) );
52 | checkEndRecord( ptr );
53 | }
54 |
55 | /**
56 | * Determines whether this CDR represents a CDF version of equal to
57 | * or greater than a given target version.
58 | *
59 | * @param targetVersion major version number to test against
60 | * @param targetRelease minor version number to test against
61 | * @return true iff this version is at least targetVersion.targetRelease
62 | */
63 | private boolean versionAtLeast( int targetVersion, int targetRelease ) {
64 | return this.version > targetVersion
65 | || this.version == targetVersion && this.release >= targetRelease;
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/RecordPlan.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 |
5 | /**
6 | * Records basic information about the position, extent and type of
7 | * a CDF record.
8 | *
9 | * @author Mark Taylor
10 | * @since 18 Jun 2013
11 | */
12 | public class RecordPlan {
13 |
14 | private final long start_;
15 | private final long recSize_;
16 | private final int recType_;
17 | private final Buf buf_;
18 |
19 | /**
20 | * Constructor.
21 | *
22 | * @param start offset into buffer of record start
23 | * @param recSize number of bytes comprising record
24 | * @param recType integer record type field
25 | * @param buf buffer containing record bytes
26 | */
27 | public RecordPlan( long start, long recSize, int recType, Buf buf ) {
28 | start_ = start;
29 | recSize_ = recSize;
30 | recType_ = recType;
31 | buf_ = buf;
32 | }
33 |
34 | /**
35 | * Returns the size of the record in bytes.
36 | *
37 | * @return record size
38 | */
39 | public long getRecordSize() {
40 | return recSize_;
41 | }
42 |
43 | /**
44 | * Returns the type code identifying what kind of CDF record it is.
45 | *
46 | * @return record type
47 | */
48 | public int getRecordType() {
49 | return recType_;
50 | }
51 |
52 | /**
53 | * Returns the buffer containing the record data.
54 | *
55 | * @return buffer
56 | */
57 | public Buf getBuf() {
58 | return buf_;
59 | }
60 |
61 | /**
62 | * Returns a pointer initially pointing at the first content byte of
63 | * the record represented by this plan.
64 | * This is the first item after the RecordSize and RecordType items
65 | * that always appear first in a CDF record, and whose values are
66 | * known by this object.
67 | *
68 | * @return pointer pointing at the start of the record-type-specific
69 | * content
70 | */
71 | public Pointer createContentPointer() {
72 | long pos = start_;
73 | pos += buf_.isBit64() ? 8 : 4; // record size
74 | pos += 4; // record type
75 | return new Pointer( pos );
76 | }
77 |
78 | /**
79 | * Returns the number of bytes in this record read (or skipped) by the
80 | * current state of a given pointer.
81 | *
82 | * @param ptr pointer
83 | * @return number of bytes between record start and pointer value
84 | */
85 | public long getReadCount( Pointer ptr ) {
86 | return ptr.get() - start_;
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/omni/scripts/waveletOmniMSA.scala:
--------------------------------------------------------------------------------
1 | import breeze.linalg.DenseVector
2 | import io.github.mandar2812.PlasmaML.omni.OmniMSA.Features
3 | import io.github.mandar2812.PlasmaML.omni._
4 | import io.github.mandar2812.dynaml.DynaMLPipe
5 | import io.github.mandar2812.dynaml.analysis.VectorField
6 | import io.github.mandar2812.dynaml.kernels._
7 | import io.github.mandar2812.dynaml.models.neuralnets._
8 | import io.github.mandar2812.dynaml.pipes.{DataPipe, Encoder}
9 |
10 | OmniMSA.quietTimeSegment = ("2014/01/01/20", "2014/01/06/20")
11 | OmniMultiOutputModels.exogenousInputs = List(24,16)
12 | val numVars = OmniMultiOutputModels.exogenousInputs.length + 1
13 | OmniMultiOutputModels.orderFeat = 3
14 | OmniMultiOutputModels.orderTarget = 2
15 |
16 | val num_features = if(OmniMultiOutputModels.deltaT.isEmpty) {
17 | (1 until numVars).map(_ => math.pow(2.0, OmniMultiOutputModels.orderFeat)).sum.toInt +
18 | math.pow(2d, OmniMultiOutputModels.orderFeat).toInt
19 | } else {
20 | OmniMultiOutputModels.deltaT.sum
21 | }
22 |
23 | DstMSAExperiment.gridSize = 2
24 | DstMSAExperiment.gridStep = 0.75
25 | DstMSAExperiment.useLogScale = true
26 | DstMSAExperiment.globalOpt = "CSA"
27 | DstMSAExperiment.it = 15
28 |
29 | implicit val ev = VectorField(num_features)
30 |
31 | val gsm_encoder = Encoder(
32 | (config: Map[String, Double]) => (DenseVector.fill[Double](num_features)(config("c")), DenseVector.fill[Double](num_features)(config("s"))),
33 | (centerAndScale: (DenseVector[Double], DenseVector[Double])) => Map("c" -> centerAndScale._1(0), "s"-> centerAndScale._2(0))
34 | )
35 |
36 | val (center, scale) = gsm_encoder(Map("c" -> 1.5, "s" -> 10.5))
37 |
38 | val gsm = GaussianSpectralKernel[DenseVector[Double]](center, scale, gsm_encoder)
39 |
40 | val cubicSplineKernel = new CubicSplineKernel[DenseVector[Double]](1.5)
41 |
42 | val d = new DiracKernel(0.05)
43 | d.block_all_hyper_parameters
44 |
45 | val tKernel = new TStudentKernel(0.01/*0.5+1.0/num_features*/)
46 | tKernel.block_all_hyper_parameters
47 |
48 | val mlpKernel = new MLPKernel(1.2901485870065708, 73.92009461973996)
49 |
50 |
51 | val (model, scales) = OmniMSA.train(
52 | mlpKernel+cubicSplineKernel+tKernel, d, 2, 0.5, false,
53 | DstMSAExperiment.globalOpt, DstMSAExperiment.it,
54 | DataPipe((x: Features) => DenseVector.vertcat(x, (x*x.t).toDenseVector, DenseVector(1d))))
55 |
56 | val metricsMT = DstMSAExperiment(mlpKernel+cubicSplineKernel+gsm, d, 3, 2, useWavelets = false)
57 |
58 | metricsMT.print
59 |
60 | OmniMultiOutputModels.exogenousInputs = List(24, 16)
61 |
62 | OmniMultiOutputModels.neuronCounts = List(12, 8, 6)
63 | OmniMultiOutputModels.activations = List(MagicSELU, MagicSELU, MagicSELU, VectorLinear)
64 |
65 | DstMSAExperiment.learningRate = 0.001
66 | DstMSAExperiment.momentum = 0.005
67 | DstMSAExperiment.it = 7000
68 | DstMSAExperiment.reg = 0.0000001
69 |
70 | val metricsW = DstMSAExperiment(4, 2)
71 |
--------------------------------------------------------------------------------
/vanAllen/src/main/scala/io/github/mandar2812/PlasmaML/vanAllen/CRRESData.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.vanAllen
2 |
3 | import io.github.mandar2812.PlasmaML.SPDFData
4 | import io.github.mandar2812.dynaml.pipes.{DataPipe, IterableDataPipe}
5 | import io.github.mandar2812.dynaml.{DynaMLPipe, utils}
6 | import org.apache.log4j.Logger
7 |
8 | import scala.io.Source
9 |
10 | /**
11 | * @author mandar2812 date: 30/5/16.
12 | *
13 | * Singleton object to extract CRRES data
14 | * from NASA SPDF in the form of CDF format
15 | * files.
16 | */
17 | object CRRESData {
18 |
19 | val logger = Logger.getLogger(this.getClass)
20 |
21 | val crres_uri = "pub/data/crres/particle_mea/mea_h0_cdaweb/"
22 |
23 | /**
24 | * Download a year of CRRES data to the local disk.
25 | *
26 | * @param year The year: 1990 or 1991
27 | * @param dataRoot The location on the disk to store the downloaded files
28 | *
29 | * */
30 | def download(year: Int = 1990, dataRoot: String = "data/") = {
31 | assert(
32 | Seq(1990, 1991) contains year,
33 | "CRRES data is available only for years 1990-1991 A.D.")
34 |
35 | val ftpIndexProcess = DataPipe((s: String) => s.split("\\n").toStream) >
36 | DynaMLPipe.replaceWhiteSpaces >
37 | IterableDataPipe((line: String) => line.split(",").last) >
38 | IterableDataPipe((fileStr: String) => fileStr.contains(".cdf")) >
39 | IterableDataPipe((fileStr: String) => SPDFData.nasa_spdf_baseurl + crres_uri + year+ "/" + fileStr) >
40 | IterableDataPipe((url: String) => {
41 | val filename = url.split("/").last
42 | logger.info("Downloading file: "+filename)
43 | utils.downloadURL(url, dataRoot + filename)
44 | })
45 |
46 | //Since Jsoup does not support ftp protocols, we use the scala.io.Source object.
47 | logger.info("Getting file index from: "+SPDFData.nasa_spdf_baseurl + crres_uri + year + "/")
48 | ftpIndexProcess(Source.fromURL(SPDFData.nasa_spdf_baseurl + crres_uri + year + "/").mkString)
49 | }
50 |
51 | /**
52 | * Display help message
53 | *
54 | * */
55 | def help(topic: String = "about") = topic match {
56 | case "about" =>
57 | println("CRRES mission data download service: ")
58 | println("----------------------------------------------------------------------------")
59 | println("The CRRES mission was launched by NASA in 1990 to "+
60 | "collect information about the Earth's radiation belt. "+
61 | "The CRRES data is a less detailed version of the Van Allen radiation probes data sets.")
62 | println("Type CRRESData.help(\"usage\") for more information.")
63 | case "usage" =>
64 | println("To download: Use CRRESData.download(year, location)")
65 | println("Where year must be an integer (1990 or 1991) and "+
66 | "location specifies the fully qualified path to dump the data files into.")
67 | }
68 |
69 | }
70 |
--------------------------------------------------------------------------------
/omni/scripts/rnnCompare.R:
--------------------------------------------------------------------------------
1 | #! /usr/bin/Rscript
2 |
3 | library(RSNNS)
4 |
5 | args <- commandArgs(trailingOnly = TRUE)
6 | year <- args[1]
7 | setwd("../../data/")
8 | prefix <- "omni2_"
9 |
10 | df <- read.csv(paste(prefix, year, ".dat", sep = ""),
11 | header = FALSE, stringsAsFactors = FALSE,
12 | colClasses = rep("numeric",55),
13 | na.strings = c("99", "999.9",
14 | "9999.", "9.999", "99.99",
15 | "9999", "999999.99",
16 | "99999.99", "9999999."))
17 |
18 | trainFeatures <- df[1:7000,c(17, 22, 25)]
19 | trainTargets <- df[1:7000,41]
20 |
21 | testFeatures <- df[7001:8760,c(17, 22, 25)]
22 | testTargets <- df[7001:8760,41]
23 | tr <- normalizeData(trainFeatures, type = "0_1")
24 | te <- normalizeData(testFeatures, attr(tr, "normParams"))
25 | trL <- normalizeData(trainTargets, type = "0_1")
26 | teL <- normalizeData(testTargets, attr(trL, "normParams"))
27 |
28 | colnames(tr) <- c("Bz", "SigmaBz", "Vsw")
29 | colnames(trL) <- c("Dst")
30 |
31 | colnames(te) <- c("Bz", "SigmaBz", "Vsw")
32 | colnames(teL) <- c("Dst")
33 |
34 | modelJordan <- jordan(tr, trL, size = c(4),
35 | learnFuncParams = c(0.005, 1.75, 0.0005, 4), maxit = 1000,
36 | inputsTest = te,
37 | targetsTest = teL,
38 | linOut = TRUE, learnFunc = "QPTT")
39 |
40 | modelEL <- elman(tr, trL, size = c(4),
41 | learnFuncParams = c(0.005, 1.75, 0.0005, 4), maxit = 1000,
42 | inputsTest = te,
43 | targetsTest = teL,
44 | linOut = TRUE, learnFunc = "QPTT")
45 |
46 |
47 | #Now train an MLP based model
48 | #modelMLP <- mlp(tr, trL, size = c(5, 3),
49 | # learnFuncParams = c(0.005), maxit = 300,
50 | # inputsTest = te,
51 | # targetsTest = teL,
52 | # linOut = TRUE)
53 |
54 | plot(testTargets, type = 'l', main = "Dst prediction", sub = as.character(year),
55 | xlab = "Time (Hours)", ylab = "Dst")
56 |
57 |
58 | lines(denormalizeData(modelJordan$fittedTestValues, attr(trL, "normParams")), col="red")
59 | lines(denormalizeData(modelEL$fittedTestValues, attr(trL, "normParams")), col="green")
60 | #lines(denormalizeData(modelMLP$fittedTestValues, attr(trL, "normParams")), col="blue")
61 |
62 | hist(modelJordan$fittedTestValues - teL, col="lightblue", breaks=100)
63 | hist(modelMLP$fittedTestValues - teL, col="lightblue", breaks=100)
64 |
65 | #Show predictions on training set
66 | plot(trainTargets, type = 'l', main = "Dst prediction", sub = as.character(year),
67 | xlab = "Time (Hours)", ylab = "Dst")
68 |
69 |
70 | lines(denormalizeData(modelJordan$fitted.values, attr(trL, "normParams")), col="red")
71 | lines(denormalizeData(modelEL$fitted.values, attr(trL, "normParams")), col="green")
72 |
73 | #Plot Iterative Error
74 |
75 | plotIterativeError(modelEL)
76 | plotIterativeError(modelJordan)
77 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/NumericEncoding.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | /**
4 | * Enumeration of numeric encoding values supported by CDF.
5 | *
6 | * @author Mark Taylor
7 | * @since 20 Jun 2013
8 | */
9 | public enum NumericEncoding {
10 |
11 | NETWORK( Boolean.TRUE ),
12 | SUN( Boolean.TRUE ),
13 | NeXT( Boolean.TRUE ),
14 | MAC( Boolean.TRUE ),
15 | HP( Boolean.TRUE ),
16 | SGi( Boolean.TRUE ),
17 | IBMRS( Boolean.TRUE ),
18 |
19 | DECSTATION( Boolean.FALSE ),
20 | IBMPC( Boolean.FALSE ),
21 | ALPHAOSF1( Boolean.FALSE ),
22 | ALPHAVMSi( Boolean.FALSE ),
23 |
24 | VAX( null ),
25 | ALPHAVMSd( null ),
26 | ALPHAVMSg( null );
27 |
28 | private final Boolean isBigendian_;
29 |
30 | /**
31 | * Constructor.
32 | *
33 | * @param isBigendian TRUE for simple big-endian,
34 | * FALSE for simple little-endian,
35 | * null for something else
36 | */
37 | NumericEncoding( Boolean isBigendian ) {
38 | isBigendian_ = isBigendian;
39 | }
40 |
41 | /**
42 | * Gives the big/little-endianness of this encoding, if that's all
43 | * the work that has to be done.
44 | * If the return value is non-null, then numeric values are
45 | * encoded the same way that java does it (two's complement for
46 | * integers and IEEE754 for floating point) with big- or little-endian
47 | * byte ordering, according to the return value.
48 | * Otherwise, some unspecified encoding is in operation.
49 | *
50 | * @return TRUE for simple big-endian, FALSE for simple little-endian,
51 | * null for something weird
52 | */
53 | public Boolean isBigendian() {
54 | return isBigendian_;
55 | }
56 |
57 | /**
58 | * Returns the encoding corresponding to the value of the
59 | * encoding field of the CDF Descriptor Record.
60 | *
61 | * @param code encoding code
62 | * @return encoding object
63 | * @throws CdfFormatException if code is unknown
64 | */
65 | public static NumericEncoding getEncoding( int code )
66 | throws CdfFormatException {
67 | switch ( code ) {
68 | case 1: return NETWORK;
69 | case 2: return SUN;
70 | case 3: return VAX;
71 | case 4: return DECSTATION;
72 | case 5: return SGi;
73 | case 6: return IBMPC;
74 | case 7: return IBMRS;
75 | case 9: return MAC;
76 | case 11: return HP;
77 | case 12: return NeXT;
78 | case 13: return ALPHAOSF1;
79 | case 14: return ALPHAVMSd;
80 | case 15: return ALPHAVMSg;
81 | case 16: return ALPHAVMSi;
82 | default:
83 | throw new CdfFormatException( "Unknown numeric encoding "
84 | + code );
85 | }
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/DynamicRBFSWLoss.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.helios.core
2 |
3 | import org.platanios.tensorflow.api.learn.Mode
4 | import org.platanios.tensorflow.api.learn.layers.{Layer, Loss}
5 | import org.platanios.tensorflow.api._
6 | import org.platanios.tensorflow.api.ops.Output
7 |
8 | /**
9 | *
RBF-Kernel Weighted Solar Wind Loss (KSW Loss)
10 | * avec dynamic time scale prediction
11 | *
12 | * A weighted loss function which enables fuzzy learning of
13 | * the solar wind propagation from heliospheric
14 | * images to ACE.
15 | *
16 | * @author mandar2812
17 | * */
18 | case class DynamicRBFSWLoss[
19 | P: TF: IsFloatOrDouble,
20 | T: TF: IsNumeric: IsNotQuantized,
21 | L: TF : IsFloatOrDouble](
22 | override val name: String,
23 | size_causal_window: Int) extends
24 | Loss[((Output[P], Output[P]), Output[T]), L](name) {
25 |
26 | override val layerType: String = s"DynamicRBFSW[$size_causal_window]"
27 |
28 | override def forwardWithoutContext(
29 | input: ((Output[P], Output[P]), Output[T]))(
30 | implicit mode: Mode): Output[L] = {
31 |
32 | //Obtain section corresponding to velocity predictions
33 | val preds = input._1._1
34 |
35 | val times_and_scales = input._1._2
36 |
37 | val times = times_and_scales(::, 0)
38 |
39 | val timescales = times_and_scales(::, 1)
40 |
41 | val repeated_timescales = tf.stack(Seq.fill(size_causal_window)(timescales), axis = -1)
42 |
43 | val repeated_times = tf.stack(Seq.fill(size_causal_window)(times), axis = -1)
44 |
45 | val index_times = Tensor((0 until size_causal_window).map(_.toDouble))
46 | .reshape(Shape(1, size_causal_window))
47 | .castTo[P]
48 |
49 | val repeated_preds = tf.stack(Seq.fill(size_causal_window)(preds), axis = -1)
50 |
51 | val convolution_kernel =
52 | (repeated_times - index_times)
53 | .abs
54 | .multiply(Tensor(-1d).castTo[P].toOutput)
55 | .divide(repeated_timescales)
56 | .exp
57 |
58 | val weighted_loss_tensor =
59 | (repeated_preds - input._2.castTo[P])
60 | .square
61 | .multiply(convolution_kernel)
62 | .sum(axes = 1)
63 | .divide(convolution_kernel.sum(axes = 1))
64 | .mean()
65 |
66 | weighted_loss_tensor.castTo[L]
67 | }
68 | }
69 |
70 | object DynamicRBFSWLoss {
71 |
72 | def output_mapping[P: TF: IsFloatOrDouble](
73 | name: String, size_causal_window: Int): Layer[Output[P], (Output[P], Output[P])] =
74 | new Layer[Output[P], (Output[P], Output[P])](name) {
75 |
76 | private[this] val scaling = Tensor(size_causal_window.toDouble-1d).castTo[P].toOutput
77 |
78 | override val layerType: String = s"OutputDynamicRBFSW[$size_causal_window]"
79 |
80 | override def forwardWithoutContext(input: Output[P])(implicit mode: Mode): (Output[P], Output[P]) = {
81 | (input(::, 0), tf.concatenate(Seq(input(::, 1).sigmoid.multiply(scaling), input(::, 2).exp), axis = 1))
82 | }
83 | }
84 |
85 | }
86 |
--------------------------------------------------------------------------------
/helios/scripts/goes_wtloss_extreme.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.PlasmaML.helios
2 | import ammonite.ops._
3 | import io.github.mandar2812.dynaml.repl.Router.main
4 | import org.joda.time._
5 | import org.platanios.tensorflow.api._
6 | import _root_.io.github.mandar2812.dynaml.tensorflow.dtflearn
7 | import org.platanios.tensorflow.api.ops.NN.SameConvPadding
8 |
9 | @main
10 | def main(
11 | test_year: Int = 2003,
12 | longWL: Boolean = false,
13 | re: Boolean = true,
14 | tmpdir: Path = root/"home"/System.getProperty("user.name")/"tmp",
15 | resFile: String = "mdi_ext_wtloss_results.csv",
16 | maxIt: Int = 200000) = {
17 |
18 | //Data with MDI images
19 |
20 | val results = if(longWL) "longwl_"+resFile else resFile
21 |
22 | print("Running experiment with test split from year: ")
23 | pprint.pprintln(test_year)
24 |
25 | val data = helios.data.generate_data_goes()
26 |
27 | println("Starting data set created.")
28 | println("Proceeding to load images & labels into Tensors ...")
29 | val flux_threshold = if(longWL) -5.5d else -6.5d
30 |
31 | val test_start = new DateTime(test_year, 1, 1, 0, 0)
32 |
33 | val test_end = new DateTime(test_year, 12, 31, 23, 59)
34 |
35 | val tt_partition = if(longWL) {
36 |
37 | (p: (DateTime, (Path, (Double, Double)))) =>
38 | if (p._1.isAfter(test_start) && p._1.isBefore(test_end) && p._2._2._2 >= flux_threshold) false
39 | else true
40 |
41 | } else {
42 |
43 | (p: (DateTime, (Path, (Double, Double)))) =>
44 | if(p._1.isAfter(test_start) && p._1.isBefore(test_end) && p._2._2._1 >= flux_threshold) false
45 | else true
46 |
47 | }
48 |
49 | val summary_dir = if(re) "mdi_wtloss_ext_resample_"+test_year else "mdi_wtloss_ext_"+test_year
50 |
51 | val architecture = {
52 | tf.learn.Cast("Input/Cast", FLOAT32) >>
53 | dtflearn.conv2d_pyramid(2, 4)(6, 3)(0.01f, dropout = true, 0.6f) >>
54 | tf.learn.MaxPool("MaxPool_3", Seq(1, 2, 2, 1), 1, 1, SameConvPadding) >>
55 | dtflearn.conv2d_unit(Shape(2, 2, 8, 4), (16, 16), dropout = false, relu_param = 0.01f)(4) >>
56 | tf.learn.MaxPool("MaxPool_5", Seq(1, 2, 2, 1), 1, 1, SameConvPadding) >>
57 | tf.learn.Flatten("Flatten_5") >>
58 | dtflearn.feedforward(128)(6) >>
59 | dtflearn.Tanh("Tanh_6") >>
60 | dtflearn.feedforward(64)(7) >>
61 | dtflearn.Tanh("Tanh_7") >>
62 | dtflearn.feedforward(32)(8) >>
63 | dtflearn.Tanh("Tanh_6") >>
64 | tf.learn.Linear("OutputLayer", 1)
65 | }
66 |
67 | val res = helios.run_experiment_goes(
68 | data, tt_partition, resample = re,
69 | longWavelength = longWL)(
70 | summary_dir, maxIt, tmpdir,
71 | arch = architecture,
72 | lossFunc = helios.learn.weightedL2FluxLoss("Loss/WeightedL2FluxLoss"))
73 |
74 |
75 | //Write the cross validation score in a results file
76 |
77 | val accuracy = res._3.results(0).sum()
78 |
79 | if(!exists(tmpdir/results)) write(tmpdir/results, "testyear,accuracy\n")
80 |
81 | write.append(tmpdir/results, s"$test_year,$accuracy\n")
82 |
83 | pprint.pprintln(res)
84 |
85 | res
86 | }
87 |
--------------------------------------------------------------------------------
/mag-core/scripts/radialDiffusionLinearDecay.sc:
--------------------------------------------------------------------------------
1 | import breeze.linalg._
2 | import com.quantifind.charts.Highcharts._
3 | import io.github.mandar2812.PlasmaML.dynamics.diffusion.RadialDiffusion
4 |
5 | val (nL,nT) = (500, 50)
6 |
7 | val lMax = 10
8 | val tMax = 5
9 |
10 | val lShellLimits = (1.0, 10.0)
11 | val timeLimits = (0.0, 5.0)
12 |
13 | val omega = 2*math.Pi/(lShellLimits._2 - lShellLimits._1)
14 | val theta = 0.006
15 | val alpha = 0.01 + theta*math.pow(omega*lShellLimits._2, 2.0)
16 |
17 | val fl = (l: Double, _: Double) => math.sin(omega*(l - lShellLimits._1))
18 | val ft = (_: Double, t: Double) => math.exp(-alpha*t) + 1.0
19 |
20 | val referenceSolution = (l: Double, t: Double) => fl(l,t)*ft(l,t)
21 |
22 | val radialDiffusionSolver = (binsL: Int, binsT: Int) => new RadialDiffusion(lShellLimits, timeLimits, binsL, binsT)
23 |
24 | val q = (l: Double, t: Double) => 0.0//alpha*fl(l,t)
25 | val dll = (l: Double, _: Double) => theta*l*l
26 | val loss = (l: Double, _: Double) => alpha - math.pow(l*omega, 2.0)*theta
27 |
28 | val initialPSD = (l: Double) => referenceSolution(l, 0.0)
29 |
30 |
31 | val rds = new RadialDiffusion(lShellLimits, timeLimits, nL, nT)
32 |
33 | val (lShellVec, timeVec) = RadialDiffusion.buildStencil(lShellLimits, nL, timeLimits, nT)
34 |
35 | val initialPSDGT: DenseVector[Double] = DenseVector(lShellVec.map(l => referenceSolution(l, 0.0)).toArray)
36 |
37 |
38 | val solution = rds.solve(q, dll, loss)(initialPSD)
39 |
40 | /*
41 | *
42 | * First set of plots
43 | *
44 | * */
45 |
46 | spline(timeVec.zip(solution.map(_(0))))
47 | hold()
48 |
49 | (1 to lMax).foreach(l => {
50 | spline(timeVec.zip(solution.map(_(l*5))))
51 | })
52 |
53 | unhold()
54 |
55 | legend(DenseVector.tabulate[Double](lMax+1)(i =>
56 | if(i < nL) lShellLimits._1+(rds.deltaL*i*5)
57 | else lShellLimits._2).toArray.map(s => "L = "+"%3f".format(s)))
58 | title("Evolution of Phase Space Density f(L,t)")
59 | xAxis("time")
60 | yAxis("f(L,t)")
61 |
62 |
63 | /*
64 | *
65 | * Second set of plots
66 | *
67 | * */
68 | spline(lShellVec.toArray.toSeq.zip(solution.head.toArray.toSeq))
69 | hold()
70 |
71 | (1 to tMax).foreach(l => {
72 | spline(lShellVec.toArray.toSeq.zip(solution(l*10).toArray.toSeq))
73 | })
74 |
75 | unhold()
76 |
77 | legend(DenseVector.tabulate[Double](tMax+1)(i =>
78 | if(i < nL) timeLimits._1+(rds.deltaT*i*10)
79 | else timeLimits._2).toArray.map(s => "t = "+"%3f".format(s)))
80 |
81 | title("Variation of Phase Space Density f(L,t)")
82 | xAxis("L")
83 | yAxis("f(L,t)")
84 |
85 | spline(lShellVec.toArray.map(lShell => (lShell, referenceSolution(lShell, 0.0))).toSeq)
86 | hold()
87 |
88 | (1 to tMax).foreach(l => {
89 | spline(lShellVec.toArray.map(lShell => (lShell, referenceSolution(lShell, timeVec(l*10)))).toSeq)
90 | })
91 |
92 | unhold()
93 |
94 | legend(DenseVector.tabulate[Double](tMax+1)(i =>
95 | if(i < nL) timeLimits._1+(rds.deltaT*i*10)
96 | else timeLimits._2).toArray.map(s => "t = "+"%3f".format(s)))
97 | xAxis("L")
98 | yAxis("f(L,t)")
99 | title("Reference Solution")
100 |
101 |
--------------------------------------------------------------------------------
/mag-core/scripts/diffusionInjectionVerification.sc:
--------------------------------------------------------------------------------
1 | import breeze.linalg._
2 | import com.quantifind.charts.Highcharts._
3 | import com.quantifind.charts.highcharts.AxisType
4 | import io.github.mandar2812.PlasmaML.dynamics.diffusion.RadialDiffusion
5 |
6 |
7 | val (nL,nT) = (10, 10)
8 |
9 |
10 | val bins = List(1, 10, 50, 100, 500, 1000, 5000, 10000)
11 |
12 | val lShellLimits = (1.0, 10.0)
13 | val timeLimits = (0.0, 5.0)
14 |
15 | //Define parameters of reference solution
16 | val a = 2*math.Pi/(lShellLimits._2 - lShellLimits._1)
17 | val b = math.log(2d)/timeLimits._2
18 |
19 | val referenceSolution = (l: Double, t: Double) => math.sin(a*(l - lShellLimits._1))*(math.exp(b*t) - 1.0)
20 | val loss = (l: Double, t: Double) => 0.0
21 |
22 | val initialPSD = (l: Double) => referenceSolution(l, 0.0)
23 |
24 | //Define parameters of radial diffusion system
25 | val alpha = 0.9
26 | val beta = 2.0
27 |
28 | val dll = (l: Double, _: Double) => alpha*math.pow(l, beta)
29 |
30 | val q = (l: Double, t: Double) => {
31 | b*math.sin(a*(l - lShellLimits._1))*math.exp(b*t) -
32 | a*alpha*(beta-2d)*math.pow(l, beta-1d)*(math.exp(b*t) - 1.0)*math.cos(a*(l - lShellLimits._1)) +
33 | a*a*alpha*math.pow(l, beta)*(math.exp(b*t) - 1.0)*math.sin(a*(l - lShellLimits._1))
34 | }
35 |
36 | val radialDiffusionSolver = (binsL: Int, binsT: Int) => new RadialDiffusion(lShellLimits, timeLimits, binsL, binsT)
37 |
38 |
39 | //Perform verification of errors for constant nL
40 |
41 | val lossesTime = bins.map(bT => {
42 |
43 | val rds = radialDiffusionSolver(nL, bT)
44 |
45 | println("Solving for delta T = "+rds.deltaT)
46 |
47 | val (lShellVec, timeVec) = RadialDiffusion.buildStencil(lShellLimits, nL, timeLimits, bT)
48 |
49 | println("\tGenerating neural computation stack")
50 |
51 | val solution = rds.solve(q, dll, loss)(initialPSD)
52 |
53 | val referenceSol = timeVec.map(t => DenseVector(lShellVec.map(lS => referenceSolution(lS, t)).toArray))
54 |
55 | println("\tCalculating RMSE with respect to reference solution\n")
56 | val error = math.sqrt(solution.zip(referenceSol).map(c => math.pow(norm(c._1 - c._2, 2.0), 2.0)).sum/(bT+1.0))
57 |
58 | (rds.deltaT, error)
59 |
60 | })
61 |
62 |
63 | val lossesSpace = bins.map(bL => {
64 |
65 | val rds = radialDiffusionSolver(bL, nT)
66 |
67 | println("Solving for delta L = "+rds.deltaL)
68 |
69 | val (lShellVec, timeVec) = RadialDiffusion.buildStencil(lShellLimits, bL, timeLimits, nT)
70 |
71 | println("\tGenerating neural computation stack")
72 |
73 | val solution = rds.solve(q, dll, loss)(initialPSD)
74 |
75 | val referenceSol = timeVec.map(t => DenseVector(lShellVec.map(lS => referenceSolution(lS, t)).toArray))
76 |
77 | println("\tCalculating RMSE with respect to reference solution\n")
78 | val error = math.sqrt(solution.zip(referenceSol).map(c => math.pow(norm(c._1 - c._2, 2.0), 2.0)/(bL+1.0)).sum/(nT+1d))
79 |
80 | (rds.deltaL, error)
81 |
82 | })
83 |
84 |
85 | spline(lossesTime)
86 | title("Forward Solver Error")
87 | xAxisType(AxisType.logarithmic)
88 | xAxis("delta T")
89 | yAxis("RMSE")
90 |
91 |
92 | spline(lossesSpace)
93 | title("Forward Solver Error")
94 | xAxisType(AxisType.logarithmic)
95 | xAxis("delta L")
96 | yAxis("RMSE")
97 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/MQPSDBasis.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.dynaml.analysis.RadialBasis
5 | import io.github.mandar2812.dynaml.pipes.{Basis, DataPipe}
6 | import io.github.mandar2812.dynaml.analysis.implicits._
7 | import spire.algebra.InnerProductSpace
8 |
9 | /**
10 | * Phase Space Density: Multi-Quadric Mesh-Based Radial Basis
11 | *
12 | * Implements a multi-quadric radial basis expansion
13 | * for the phase space density with nodes placed on a
14 | * regular space time mesh.
15 | *
16 | * */
17 | class MQPSDBasis(
18 | lShellLimits: (Double, Double), nL: Int,
19 | timeLimits: (Double, Double), nT: Int,
20 | logScales: (Boolean, Boolean) = (false, false))
21 | extends PSDRadialBasis(
22 | lShellLimits, nL,
23 | timeLimits, nT,
24 | logScales) {
25 |
26 | private val beta = -1
27 |
28 | val activation: DataPipe[Double, Double] = RadialBasis.multiquadric
29 |
30 | val field: InnerProductSpace[(Double, Double), Double] = innerProdTuple2
31 |
32 | override protected val f: ((Double, Double)) => DenseVector[Double] =
33 | (x: (Double, Double)) => DenseVector(
34 | centers.zip(scales).map(cs => {
35 | val d = field.minus(x, cs._1)
36 | val scaledDist = (d._1/cs._2._1, d._2/cs._2._2)
37 | val r = math.sqrt(field.dot(scaledDist, scaledDist))
38 |
39 | activation(r)
40 | }).toArray
41 | )
42 |
43 | override val f_l: ((Double, Double)) => DenseVector[Double] = (x: (Double, Double)) => {
44 | DenseVector(
45 | centers.zip(scales).map(cs => {
46 | val (c, (theta_s, _)) = cs
47 |
48 | val d = field.minus(x, c)
49 |
50 | val scaledDist = d._1/theta_s
51 |
52 | val g_l = 1d + scaledDist*scaledDist
53 |
54 | val invThetaS = 1/theta_s
55 |
56 | -beta*invThetaS*math.abs(d._1)/g_l
57 | }).toArray) *:*
58 | f(x)
59 | }
60 |
61 | override val f_ll: ((Double, Double)) => DenseVector[Double] = (x: (Double, Double)) => {
62 | DenseVector(
63 | centers.zip(scales).map(cs => {
64 | val (c, (theta_s, _)) = cs
65 |
66 | val d = field.minus(x, c)
67 |
68 | val scaledDist = d._1/theta_s
69 |
70 | val g_l = 1d + scaledDist*scaledDist
71 |
72 | val invThetaS = 1/theta_s
73 |
74 | val sq = (s: Double) => s*s
75 |
76 | beta*invThetaS*(sq(d._1)*(0.5*beta+1) - g_l)/math.pow(g_l, 2+0.5*beta)
77 | }).toArray) *:*
78 | f(x)
79 | }
80 |
81 | override val f_t: ((Double, Double)) => DenseVector[Double] = (x: (Double, Double)) => {
82 |
83 | DenseVector(
84 | centers.zip(scales).map(cs => {
85 | val (c, (_, theta_t)) = cs
86 |
87 | val d = field.minus(x, c)
88 |
89 | val scaledDist = d._2/theta_t
90 |
91 | val g_t = 1d + scaledDist*scaledDist
92 |
93 | val invThetaT = 1/theta_t
94 |
95 | -beta*invThetaT*math.abs(d._2)/g_t
96 | }).toArray) *:*
97 | f(x)
98 | }
99 |
100 |
101 | }
102 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/WrapperBuf.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 |
5 | import java.io.IOException;
6 | import java.io.InputStream;
7 |
8 | /**
9 | * Buf implementation based on an existing Buf instance.
10 | * All methods are delegated to the base buf.
11 | *
12 | * @author Mark Taylor
13 | * @since 18 Jun 2013
14 | */
15 | public class WrapperBuf implements Buf {
16 |
17 | private final Buf base_;
18 |
19 | /**
20 | * Constructor.
21 | *
22 | * @param base base buf
23 | */
24 | public WrapperBuf( Buf base ) {
25 | base_ = base;
26 | }
27 |
28 | public long getLength() {
29 | return base_.getLength();
30 | }
31 |
32 | public int readUnsignedByte( Pointer ptr ) throws IOException {
33 | return base_.readUnsignedByte( ptr );
34 | }
35 |
36 | public int readInt( Pointer ptr ) throws IOException {
37 | return base_.readInt( ptr );
38 | }
39 |
40 | public long readOffset( Pointer ptr ) throws IOException {
41 | return base_.readOffset( ptr );
42 | }
43 |
44 | public String readAsciiString( Pointer ptr, int nbyte ) throws IOException {
45 | return base_.readAsciiString( ptr, nbyte );
46 | }
47 |
48 | public void setBit64( boolean bit64 ) {
49 | base_.setBit64( bit64 );
50 | }
51 |
52 | public boolean isBit64() {
53 | return base_.isBit64();
54 | }
55 |
56 | public void setEncoding( boolean isBigendian ) {
57 | base_.setEncoding( isBigendian );
58 | }
59 |
60 | public boolean isBigendian() {
61 | return base_.isBigendian();
62 | }
63 |
64 | public void readDataBytes( long offset, int count, byte[] array )
65 | throws IOException {
66 | base_.readDataBytes( offset, count, array );
67 | }
68 |
69 | public void readDataShorts( long offset, int count, short[] array )
70 | throws IOException {
71 | base_.readDataShorts( offset, count, array );
72 | }
73 |
74 | public void readDataInts( long offset, int count, int[] array )
75 | throws IOException {
76 | base_.readDataInts( offset, count, array );
77 | }
78 |
79 | public void readDataLongs( long offset, int count, long[] array )
80 | throws IOException {
81 | base_.readDataLongs( offset, count, array );
82 | }
83 |
84 | public void readDataFloats( long offset, int count, float[] array )
85 | throws IOException {
86 | base_.readDataFloats( offset, count, array );
87 | }
88 |
89 | public void readDataDoubles( long offset, int count, double[] array )
90 | throws IOException {
91 | base_.readDataDoubles( offset, count, array );
92 | }
93 |
94 | public InputStream createInputStream( long offset ) {
95 | return base_.createInputStream( offset );
96 | }
97 |
98 | public Buf fillNewBuf( long count, InputStream in ) throws IOException {
99 | return base_.fillNewBuf( count, in );
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/mag-core/scripts/radialDiffusionInjection.sc:
--------------------------------------------------------------------------------
1 | import breeze.linalg._
2 | import com.quantifind.charts.Highcharts._
3 | import io.github.mandar2812.PlasmaML.dynamics.diffusion.RadialDiffusion
4 |
5 |
6 | val (nL,nT) = (200, 50)
7 | val lMax = 10
8 | val tMax = 5
9 |
10 | val lShellLimits = (1.0, 7.0)
11 | val timeLimits = (0.0, 5.0)
12 | val a = 2*math.Pi/(lShellLimits._2 - lShellLimits._1)
13 | val b = math.log(2d)/timeLimits._2
14 |
15 | val referenceSolution = (l: Double, t: Double) => math.sin(a*(l - lShellLimits._1))*(math.exp(b*t) - 1.0)
16 |
17 | //Define parameters of radial diffusion system
18 | val alpha = 0.9
19 | val beta = 2.0
20 |
21 | val dll = (l: Double, _: Double) => alpha*math.pow(l, beta)
22 |
23 | val q = (l: Double, t: Double) => {
24 | b*math.sin(a*(l - lShellLimits._1))*math.exp(b*t) -
25 | a*alpha*(beta-2d)*math.pow(l, beta-1d)*(math.exp(b*t) - 1.0)*math.cos(a*(l - lShellLimits._1)) +
26 | a*a*alpha*math.pow(l, beta)*(math.exp(b*t) - 1.0)*math.sin(a*(l - lShellLimits._1))
27 | }
28 |
29 | val loss = (_: Double, _:Double) => 0.0
30 |
31 | val boundFlux = (l: Double, t: Double) => {
32 | if(l == lShellLimits._1 || l == lShellLimits._2) referenceSolution(l, t) else 0.0
33 | }
34 |
35 | val initialPSD = (l: Double) => referenceSolution(l, 0.0)
36 |
37 | val rds = new RadialDiffusion(lShellLimits, timeLimits, nL, nT)
38 |
39 | val (lShellVec, timeVec) = RadialDiffusion.buildStencil(lShellLimits, nL, timeLimits, nT)
40 |
41 |
42 | val initialPSDGT: DenseVector[Double] = DenseVector(lShellVec.map(l => referenceSolution(l, 0.0)).toArray)
43 |
44 |
45 | val solution = rds.solve(q, dll, loss)(initialPSD)
46 |
47 | /*
48 | *
49 | * First set of plots
50 | *
51 | * */
52 |
53 | spline(timeVec.zip(solution.map(_(0))))
54 | hold()
55 |
56 | (1 to lMax).foreach(l => {
57 | spline(timeVec.zip(solution.map(_(l*5))))
58 | })
59 |
60 | unhold()
61 |
62 | legend(
63 | DenseVector.tabulate[Double](lMax+1)(
64 | i =>
65 | if(i < nL) lShellLimits._1+(rds.deltaL*i*5)
66 | else lShellLimits._2)
67 | .toArray
68 | .map(s => "L = "+"%3f".format(s))
69 | )
70 |
71 | title("Evolution of Phase Space Density f(L,t)")
72 | xAxis("time")
73 | yAxis("f(L,t)")
74 |
75 |
76 | /*
77 | *
78 | * Second set of plots
79 | *
80 | * */
81 | spline(lShellVec.toArray.toSeq.zip(solution.head.toArray.toSeq))
82 | hold()
83 |
84 | (1 to tMax).foreach(l => {
85 | spline(lShellVec.toArray.toSeq.zip(solution(l*10).toArray.toSeq))
86 | })
87 |
88 | unhold()
89 |
90 | legend(DenseVector.tabulate[Double](tMax+1)(i =>
91 | if(i < nL) timeLimits._1+(rds.deltaT*i*10)
92 | else timeLimits._2).toArray.map(s => "t = "+"%3f".format(s)))
93 |
94 | title("Variation of Phase Space Density f(L,t)")
95 | xAxis("L")
96 | yAxis("f(L,t)")
97 |
98 | spline(lShellVec.toArray.map(lShell => (lShell, referenceSolution(lShell, 0.0))).toSeq)
99 | hold()
100 |
101 | (1 to tMax).foreach(l => {
102 | spline(lShellVec.toArray.map(lShell => (lShell, referenceSolution(lShell, timeVec(l*10)))).toSeq)
103 | })
104 |
105 | unhold()
106 |
107 | legend(DenseVector.tabulate[Double](tMax+1)(i =>
108 | if(i < nL) timeLimits._1+(rds.deltaT*i*10)
109 | else timeLimits._2).toArray.map(s => "t = "+"%3f".format(s)))
110 | xAxis("L")
111 | yAxis("f(L,t)")
112 | title("Reference Solution")
113 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/GaussianPSDBasis.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.dynaml.analysis.RadialBasis
5 | import io.github.mandar2812.dynaml.analysis.implicits._
6 | import io.github.mandar2812.dynaml.pipes._
7 |
8 | /**
9 | * Phase Space Density: Gaussian Mesh-Based Radial Basis
10 | *
11 | * Implements a gaussian radial basis expansion for the phase space density
12 | * with nodes placed on a regular space time mesh.
13 | *
14 | * */
15 | class GaussianPSDBasis(
16 | lShellLimits: (Double, Double),
17 | nL: Int,
18 | timeLimits: (Double, Double),
19 | nT: Int,
20 | logScales: (Boolean, Boolean) = (false, false))
21 | extends PSDRadialBasis(lShellLimits, nL, timeLimits, nT, logScales) {
22 |
23 | val (basisL, basisT) = (
24 | RadialBasis.gaussianBasis(lSeq, scalesL, bias = false),
25 | RadialBasis.gaussianBasis(tSeq, scalesT, bias = false)
26 | )
27 |
28 | val productBasis: Basis[(Double, Double)] = basisL * basisT
29 |
30 | override protected val f: ((Double, Double)) => DenseVector[Double] =
31 | (x: (Double, Double)) => productBasis(x)
32 |
33 | override val f_l: ((Double, Double)) => DenseVector[Double] =
34 | (x: (Double, Double)) => {
35 | val (l, _) = x
36 |
37 | DenseVector(
38 | centers
39 | .zip(scales)
40 | .map(cs => {
41 | val ((l_tilda, _), (theta_s, theta_t)) = cs
42 | val grL = (i: Int, j: Int) => gradSqNormDouble(i, j)(l, l_tilda)
43 |
44 | val sq = (s: Double) => s * s
45 |
46 | val (invThetaS, _) = (sq(1 / theta_s), sq(1 / theta_t))
47 |
48 | 0.5 * invThetaS * grL(1, 0)
49 | })
50 | .toArray
51 | ) *:*
52 | f(x)
53 | }
54 |
55 | override val f_ll: ((Double, Double)) => DenseVector[Double] =
56 | (x: (Double, Double)) => {
57 | val (l, _) = x
58 |
59 | DenseVector(
60 | centers
61 | .zip(scales)
62 | .map(cs => {
63 | val ((l_tilda, _), (theta_s, theta_t)) = cs
64 |
65 | val grL = (i: Int, j: Int) => gradSqNormDouble(i, j)(l, l_tilda)
66 | val sq = (s: Double) => s * s
67 |
68 | val (invThetaS, _) = (sq(1 / theta_s), sq(1 / theta_t))
69 |
70 | val gs = 0.5 * invThetaS * sq(grL(1, 0)) - grL(2, 0)
71 |
72 | 0.5 * invThetaS * gs
73 | })
74 | .toArray
75 | ) *:*
76 | f(x)
77 | }
78 |
79 | override val f_t: ((Double, Double)) => DenseVector[Double] =
80 | (x: (Double, Double)) => {
81 | val (_, t) = x
82 |
83 | DenseVector(
84 | centers
85 | .zip(scales)
86 | .map(cs => {
87 | val ((_, t_tilda), (theta_s, theta_t)) = cs
88 |
89 | val grT = (i: Int, j: Int) => gradSqNormDouble(i, j)(t, t_tilda)
90 |
91 | val sq = (s: Double) => s * s
92 |
93 | val (_, invThetaT) = (sq(1 / theta_s), sq(1 / theta_t))
94 |
95 | 0.5 * invThetaT * grT(1, 0)
96 | })
97 | .toArray
98 | ) *:*
99 | f(x)
100 | }
101 |
102 | }
103 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/util/LogUtil.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.util;
2 |
3 | import java.util.logging.ConsoleHandler;
4 | import java.util.logging.Formatter;
5 | import java.util.logging.Handler;
6 | import java.util.logging.Level;
7 | import java.util.logging.LogRecord;
8 | import java.util.logging.Logger;
9 |
10 | /**
11 | * Utilities for controlling logging level.
12 | *
13 | * @author Mark Taylor
14 | * @since 21 Jun 2013
15 | */
16 | public class LogUtil {
17 |
18 | /**
19 | * Private constructor prevents instantiation.
20 | */
21 | private LogUtil() {
22 | }
23 |
24 | /**
25 | * Sets the logging verbosity of the root logger and ensures that
26 | * logging messages at that level are reported to the console.
27 | * You'd think this would be simple, but it requires jumping through hoops.
28 | *
29 | * @param verbose 0 for normal, positive for more, negative for less
30 | * (0=INFO, +1=CONFIG, -1=WARNING)
31 | */
32 | public static void setVerbosity( int verbose ) {
33 |
34 | // Set a level based on the given verbosity.
35 | int ilevel = Level.INFO.intValue() - ( verbose * 100 );
36 | Level level = Level.parse( Integer.toString( ilevel ) );
37 |
38 | // Set the root logger's level to this value.
39 | Logger rootLogger = Logger.getLogger( "" );
40 | rootLogger.setLevel( level );
41 |
42 | // Make sure that the root logger's console handler will actually
43 | // emit these messages. By default it seems that anything below
44 | // INFO is squashed.
45 | Handler[] rootHandlers = rootLogger.getHandlers();
46 | if ( rootHandlers.length > 0 &&
47 | rootHandlers[ 0 ] instanceof ConsoleHandler ) {
48 | rootHandlers[ 0 ].setLevel( level );
49 | rootHandlers[ 0 ].setFormatter( new LineFormatter( false ) );
50 | }
51 | for ( int i = 0; i < rootHandlers.length; i++ ) {
52 | rootHandlers[ i ].setLevel( level );
53 | }
54 | }
55 |
56 | /**
57 | * Compact log record formatter. Unlike the default
58 | * {@link java.util.logging.SimpleFormatter} this generally uses only
59 | * a single line for each record.
60 | */
61 | public static class LineFormatter extends Formatter {
62 |
63 | private final boolean debug_;
64 |
65 | /**
66 | * Constructor.
67 | *
68 | * @param debug iff true, provides more information per log message
69 | */
70 | public LineFormatter( boolean debug ) {
71 | debug_ = debug;
72 | }
73 |
74 | public String format( LogRecord record ) {
75 | StringBuffer sbuf = new StringBuffer();
76 | sbuf.append( record.getLevel().toString() )
77 | .append( ": " )
78 | .append( formatMessage( record ) );
79 | if ( debug_ ) {
80 | sbuf.append( ' ' )
81 | .append( '(' )
82 | .append( record.getSourceClassName() )
83 | .append( '.' )
84 | .append( record.getSourceMethodName() )
85 | .append( ')' );
86 | }
87 | sbuf.append( '\n' );
88 | return sbuf.toString();
89 | }
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/helios/scripts/csss_tuning.sc:
--------------------------------------------------------------------------------
1 | repl.sess.save()
2 |
3 |
4 | import _root_.io.github.mandar2812.dynaml.{utils => dutils}
5 | import $exec.helios.scripts.csss
6 | import $exec.helios.scripts.csss_pdt_model_tuning
7 | import $exec.helios.scripts.csss_so_tuning
8 | import $exec.helios.scripts.env
9 |
10 | val time_window = (48, 72)
11 |
12 | val avg_sw_6h = DataPipe(
13 | (xs: Seq[Double]) => xs.grouped(6).map(g => g.sum / g.length).toSeq
14 | )
15 |
16 | val max_sw_6h = DataPipe(
17 | (xs: Seq[Double]) => xs.grouped(6).map(g => g.max).toSeq
18 | )
19 |
20 | val median_sw_6h = DataPipe(
21 | (xs: Seq[Double]) => xs.grouped(6).map(g => dutils.median(g.toStream)).toSeq
22 | )
23 |
24 |
25 | val avg_sw_24h = DataPipe(
26 | (xs: Seq[Double]) => xs.grouped(24).map(g => g.sum / g.length).toSeq
27 | )
28 |
29 | val max_sw_24h = DataPipe(
30 | (xs: Seq[Double]) => xs.grouped(24).map(g => g.max).toSeq
31 | )
32 |
33 | val median_sw_24h = DataPipe(
34 | (xs: Seq[Double]) => xs.grouped(24).map(g => dutils.median(g.toStream)).toSeq
35 | )
36 |
37 | val fact = 10
38 | val base_iterations = 80000
39 | def ext_iterations = base_iterations * fact
40 |
41 | val base_it_pdt = 3
42 | def ext_it_pdt = base_it_pdt + 2
43 |
44 | val csss_exp = csss_pdt_model_tuning(
45 | start_year = 2008,
46 | end_year = 2016,
47 | test_year = 2008,
48 | test_month = 12,
49 | test_rotation = Some(2077),
50 | crop_latitude = 90d,
51 | sw_threshold = 600d,
52 | fraction_pca = 1d,
53 | fte_step = 0,
54 | history_fte = 0,
55 | log_scale_omni = false,
56 | log_scale_fte = true,
57 | time_window = time_window,
58 | ts_transform_output = median_sw_6h,
59 | network_size = Seq(50, 50),
60 | use_persistence = true,
61 | activation_func = (i: Int) => timelag.utils.getReLUAct3[Double](1, 1, i, 0f),
62 | hyper_optimizer = "gs",
63 | num_samples = 4,
64 | quantity = OMNIData.Quantities.V_SW,
65 | omni_ext = Seq(
66 | OMNIData.Quantities.sunspot_number,
67 | OMNIData.Quantities.F10_7
68 | ),
69 | reg_type = "L2",
70 | batch_size = 128,
71 | max_iterations = ext_iterations,
72 | max_iterations_tuning = base_iterations,
73 | pdt_iterations_tuning = base_it_pdt,
74 | pdt_iterations_test = ext_it_pdt,
75 | optimization_algo = tf.train.Adam(0.001f),
76 | summary_dir = env.summary_dir,
77 | get_training_preds = true,
78 | data_scaling = "hybrid",
79 | use_copula = true,
80 | existing_exp = None
81 | )
82 |
83 | try {
84 | %%(
85 | 'Rscript,
86 | script,
87 | csss_exp.results.summary_dir,
88 | csss.scatter_plots_test(csss_exp.results.summary_dir).last,
89 | "test_"
90 | )
91 | } catch {
92 | case e: Exception => e.printStackTrace()
93 | }
94 |
95 | helios.visualise_cdt_results(
96 | csss.scatter_plots_test(csss_exp.results.summary_dir).last
97 | )
98 |
99 | val csss_fixed = csss_so_tuning.baseline(
100 | csss_exp,//env.summary_dir/exps(8) //exp_dir,//csss_exp.results.summary_dir,
101 | network_size = Seq(50, 50),
102 | activation_func = (i: Int) => timelag.utils.getReLUAct3[Double](1, 1, i, 0f),
103 | optimization_algo = tf.train.Adam(0.001f),
104 | max_iterations = ext_iterations,
105 | max_iterations_tuning = base_iterations,
106 | batch_size = 128,
107 | num_samples = 4,
108 | data_scaling = "hybrid",
109 | use_copula = true
110 | )
111 |
--------------------------------------------------------------------------------
/helios/scripts/pdt_const_v_tuning.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.pipes._
2 | import _root_.io.github.mandar2812.dynaml.repl.Router.main
3 | import org.platanios.tensorflow.api.ops.training.optimizers.Optimizer
4 | import _root_.io.github.mandar2812.PlasmaML.helios
5 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
6 | import _root_.ammonite.ops._
7 | import org.platanios.tensorflow.api._
8 | import org.platanios.tensorflow.api.learn.layers.Activation
9 | import org.platanios.tensorflow.api.learn.layers.Layer
10 |
11 | import $file.run_model_tuning_pdt
12 |
13 | @main
14 | def main(
15 | d: Int = 3,
16 | confounding: Seq[Double] = Seq(0d, 0.25, 0.5, 0.75),
17 | size_training: Int = 100,
18 | size_test: Int = 50,
19 | sliding_window: Int = 15,
20 | noise: Double = 0.5,
21 | noiserot: Double = 0.1,
22 | alpha: Double = 0.0,
23 | train_test_separate: Boolean = false,
24 | num_neurons: Seq[Int] = Seq(40),
25 | activation_func: Int => Layer[Output[Double], Output[Double]] =
26 | timelag.utils.getReLUAct[Double](1, _),
27 | iterations: Int = 150000,
28 | iterations_tuning: Int = 20000,
29 | pdt_iterations: Int = 2,
30 | pdt_iterations_tuning: Int = 4,
31 | miniBatch: Int = 32,
32 | optimizer: Optimizer = tf.train.AdaDelta(0.01f),
33 | sum_dir_prefix: String = "const_v",
34 | summaries_top_dir: Path = home / 'tmp,
35 | num_samples: Int = 20,
36 | hyper_optimizer: String = "gs",
37 | hyp_opt_iterations: Option[Int] = Some(5),
38 | epochFlag: Boolean = false,
39 | regularization_types: Seq[String] = Seq("L2"),
40 | checkpointing_freq: Int = 4
41 | ): Seq[timelag.ExperimentResult[Double, Double, timelag.TunedModelRun[
42 | Double,
43 | Double
44 | ]]] = {
45 |
46 | //Output computation
47 | val beta = 100f
48 |
49 | //Time Lag Computation
50 | // distance/velocity
51 | val distance = beta * 10
52 |
53 | val compute_v = DataPipe[Tensor[Double], Float](
54 | (v: Tensor[Double]) =>
55 | v.square.mean().scalar.asInstanceOf[Float] * beta / d + 100f
56 | )
57 |
58 | val compute_output: DataPipe[Tensor[Double], (Float, Float)] = DataPipe(
59 | (x: Tensor[Double]) => {
60 |
61 | val out = compute_v(x)
62 |
63 | val noisy_output = out + scala.util.Random.nextGaussian().toFloat
64 |
65 | (distance / out, noisy_output)
66 | }
67 | )
68 |
69 | val experiment_results = run_model_tuning_pdt(
70 | compute_output,
71 | d,
72 | confounding,
73 | size_training,
74 | size_test,
75 | sliding_window,
76 | noise,
77 | noiserot,
78 | alpha,
79 | train_test_separate,
80 | num_neurons,
81 | activation_func,
82 | iterations,
83 | iterations_tuning,
84 | pdt_iterations,
85 | pdt_iterations_tuning,
86 | miniBatch,
87 | optimizer,
88 | sum_dir_prefix,
89 | summaries_top_dir,
90 | num_samples,
91 | hyper_optimizer,
92 | hyp_opt_iterations,
93 | epochFlag,
94 | regularization_types,
95 | checkpointing_freq
96 | )
97 |
98 | experiment_results.map(
99 | experiment_result =>
100 | experiment_result
101 | .copy[Double, Double, timelag.TunedModelRun[Double, Double]](
102 | config = experiment_result.config
103 | .copy[Double](output_mapping = Some(compute_v))
104 | )
105 | )
106 | }
107 |
--------------------------------------------------------------------------------
/helios/scripts/visualize_predictors.R:
--------------------------------------------------------------------------------
1 | #/usr/bin/Rscript
2 | library(ggplot2)
3 | library(reshape2)
4 | library(latex2exp)
5 | library(directlabels)
6 | library(plyr)
7 |
8 | args <- commandArgs(trailingOnly = TRUE)
9 | direc <- args[1]
10 |
11 | if (length(args) > 1) {
12 | iden <- args[2]
13 | } else {
14 | ""
15 | }
16 |
17 |
18 | setwd(direc)
19 |
20 | features_train_data <- read.csv("train_data_features.csv")
21 | targets_train_data <- read.csv("train_data_targets.csv")
22 |
23 | # Get Scaling properties
24 | cx <- sapply(features_train_data, mean)
25 | sx <- sapply(features_train_data, sd)
26 |
27 | rescale_features <- function(x) {
28 | x*sx + cx
29 | }
30 |
31 | cy <- sapply(targets_train_data, mean)
32 | sy <- sapply(targets_train_data, sd)
33 |
34 | rescale_targets <- function(x) {
35 | x*sy + cy
36 | }
37 |
38 |
39 |
40 | train_split_features <- as.data.frame(sapply(read.csv("train_split_features.csv"), rescale_features))
41 |
42 | x_2 <- apply(train_split_features, c(1), function(x) sum(x ^ 2))
43 |
44 | train_split_targets <- as.data.frame(sapply(read.csv("train_split_targets.csv"), rescale_targets))
45 |
46 | selected_preds <- Filter(function(x) x%%3 == 0, 1:ncol(train_split_targets))
47 |
48 |
49 | pred_it_files <- list.files(pattern = "train_split_pdtit_[1-9]*_predictions.csv", recursive = FALSE)
50 |
51 | l <- lapply(pred_it_files, read.csv)
52 |
53 |
54 |
55 | selected_cols <- c(
56 | "x_2",
57 | sapply(selected_preds, function(x) paste(c("Target", as.character(x)), collapse = '_')),
58 | sapply(selected_preds, function(x) paste(c("Pred", as.character(x)), collapse = '_')),
59 | "it"
60 | )
61 |
62 | proc_preds <- function(index) {
63 | df <- as.data.frame(sapply(l[[index]], rescale_targets))
64 | df$it <- rep(index, nrow(df))
65 | proc <- cbind(x_2, train_split_targets, df)
66 | colnames(proc) <- c(
67 | "x_2",
68 | lapply(1:ncol(train_split_targets), function(i){paste(c("Target", i - 1), collapse = '_')}),
69 | lapply(1:ncol(train_split_targets), function(i){paste(c("Pred", i - 1), collapse = '_')}),
70 | "it"
71 | )
72 | proc[,as.vector(selected_cols)]
73 | }
74 |
75 | train_preds_and_targets_by_it <- ldply(lapply(seq_along(l), proc_preds), data.frame)
76 |
77 | scatter_predictors <- melt(
78 | train_preds_and_targets_by_it,
79 | id = c("x_2", "it")
80 | )
81 |
82 | scatter_predictors$timelag <- as.factor(
83 | unlist(
84 | lapply(
85 | as.character(scatter_predictors$variable),
86 | function(v) as.integer(strsplit(v, '_')[[1]][2])
87 | )
88 | )
89 | )
90 |
91 | scatter_predictors$variable <- as.factor(
92 | unlist(
93 | lapply(
94 | as.character(scatter_predictors$variable),
95 | function(v) as.character(strsplit(v, '_')[[1]][1])
96 | )
97 | )
98 | )
99 |
100 | palette1 <- c("#000000", "firebrick3")
101 |
102 | ggplot(
103 | scatter_predictors,
104 | aes(x = x_2, y = value, colour = variable)) +
105 | geom_point(alpha = 0.65) +
106 | theme_gray(base_size = 14) +
107 | scale_colour_manual(values=palette1, labels = unname(TeX(c('$\\hat{y}(\\mathbf{x})$', '$y(\\mathbf{x})$')))) +
108 | facet_grid(it~timelag) +
109 | xlab(TeX('$||\\mathbf{x}||_2$')) + ylab(TeX('$\\hat{y}(\\mathbf{x}) \\ y(\\mathbf{x})$'))
110 |
111 | ggsave(paste(iden, "predictors_scatter_it.pdf", sep = ''), scale = 2.5, device = pdf())
112 |
113 |
114 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/record/AttributeEntryDescriptorRecord.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf.record;
2 |
3 | import io.github.mandar2812.PlasmaML.cdf.Buf;
4 | import io.github.mandar2812.PlasmaML.cdf.CdfField;
5 | import io.github.mandar2812.PlasmaML.cdf.OffsetField;
6 |
7 | import java.io.IOException;
8 |
9 | /**
10 | * Abstract superclass for CDF Attribute Entry Descriptor Records.
11 | * Two concrete subclasses exist for AzEDRs and AgrEDRs.
12 | *
13 | * @author Mark Taylor
14 | * @since 19 Jun 2013
15 | */
16 | public abstract class AttributeEntryDescriptorRecord extends Record {
17 |
18 | @CdfField
19 | @OffsetField
20 | public final long aedrNext;
21 | @CdfField public final int attrNum;
22 | @CdfField public final int dataType;
23 | @CdfField public final int num;
24 | @CdfField public final int numElems;
25 | @CdfField public final int rfuA;
26 | @CdfField public final int rfuB;
27 | @CdfField public final int rfuC;
28 | @CdfField public final int rfuD;
29 | @CdfField public final int rfuE;
30 | private final long valueOffset_;
31 |
32 | /**
33 | * Constructor.
34 | *
35 | * @param plan basic record info
36 | * @param abbrev abbreviated name for record type
37 | * @param recordType record type code
38 | */
39 | private AttributeEntryDescriptorRecord( RecordPlan plan, String abbrev,
40 | int recordType )
41 | throws IOException {
42 | super( plan, abbrev, recordType );
43 | Buf buf = plan.getBuf();
44 | Pointer ptr = plan.createContentPointer();
45 | this.aedrNext = buf.readOffset( ptr );
46 | this.attrNum = buf.readInt( ptr );
47 | this.dataType = buf.readInt( ptr );
48 | this.num = buf.readInt( ptr );
49 | this.numElems = buf.readInt( ptr );
50 | this.rfuA = checkIntValue( buf.readInt( ptr ), 0 );
51 | this.rfuB = checkIntValue( buf.readInt( ptr ), 0 );
52 | this.rfuC = checkIntValue( buf.readInt( ptr ), 0 );
53 | this.rfuD = checkIntValue( buf.readInt( ptr ), -1 );
54 | this.rfuE = checkIntValue( buf.readInt( ptr ), -1 );
55 | valueOffset_ = ptr.get();
56 | }
57 |
58 | /**
59 | * Returns the file offset at which this record's Value field starts.
60 | *
61 | * @return file offset of Value field
62 | */
63 | public long getValueOffset() {
64 | return valueOffset_;
65 | }
66 |
67 | /**
68 | * Field data for CDF record of type Attribute g/rEntry Descriptor Record.
69 | */
70 | public static class GrVariant extends AttributeEntryDescriptorRecord {
71 |
72 | /**
73 | * Constructor.
74 | *
75 | * @param plan basic record information
76 | */
77 | public GrVariant( RecordPlan plan ) throws IOException {
78 | super( plan, "AgrEDR", 5 );
79 | }
80 | }
81 |
82 | /**
83 | * Field data for CDF record of type Attribute zEntry Descriptor Record.
84 | */
85 | public static class ZVariant extends AttributeEntryDescriptorRecord {
86 |
87 | /**
88 | * Constructor.
89 | *
90 | * @param plan basic record information
91 | */
92 | public ZVariant( RecordPlan plan ) throws IOException {
93 | super( plan, "AzEDR", 9 );
94 | }
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/mag-core/scripts/radialDiffusionVerification.sc:
--------------------------------------------------------------------------------
1 | import breeze.linalg._
2 | import com.quantifind.charts.Highcharts._
3 | import com.quantifind.charts.highcharts.AxisType
4 | import io.github.mandar2812.PlasmaML.dynamics.diffusion.RadialDiffusion
5 |
6 |
7 | val (nL,nT) = (10, 10)
8 |
9 |
10 | val bins = List(1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000)
11 |
12 | val lShellLimits = (1.0, 10.0)
13 | val timeLimits = (0.0, 5.0)
14 |
15 | val omega = 2*math.Pi/(lShellLimits._2 - lShellLimits._1)
16 | val theta = 0.006
17 | val alpha = 0.01 + theta*math.pow(omega*lShellLimits._2, 2.0)
18 |
19 | val fl = (l: Double, _: Double) => math.sin(omega*(l - lShellLimits._1))
20 | val ft = (_: Double, t: Double) => math.exp(-alpha*t) + 1.0
21 |
22 | val referenceSolution = (l: Double, t: Double) => fl(l,t)*ft(l,t)
23 |
24 | val radialDiffusionSolver = (binsL: Int, binsT: Int) => new RadialDiffusion(lShellLimits, timeLimits, binsL, binsT)
25 |
26 | val q = (l: Double, t: Double) => alpha*fl(l,t)
27 | val dll = (l: Double, _: Double) => theta*l*l
28 | val loss = (l: Double, _: Double) => alpha - math.pow(l*omega, 2.0)*theta
29 |
30 | val initialPSD = (l: Double) => referenceSolution(l, 0.0)
31 |
32 |
33 | //Perform verification of errors for constant nL
34 |
35 | val lossesTime = bins.map(bT => {
36 |
37 | val rds = radialDiffusionSolver(nL, bT)
38 |
39 | println("Solving for delta T = "+rds.deltaT)
40 |
41 | val lShellVec = DenseVector.tabulate[Double](nL+1)(i =>
42 | if(i < nL) lShellLimits._1+(rds.deltaL*i)
43 | else lShellLimits._2).toArray.toSeq
44 |
45 | val timeVec = DenseVector.tabulate[Double](bT+1)(i =>
46 | if(i < bT) timeLimits._1+(rds.deltaT*i)
47 | else timeLimits._2).toArray.toSeq
48 |
49 | println("\tGenerating neural computation stack & computing solution")
50 |
51 | val solution = rds.solve(q, dll, loss)(initialPSD)
52 |
53 | val referenceSol = timeVec.map(t =>
54 | DenseVector(lShellVec.map(lS => referenceSolution(lS, t)).toArray))
55 |
56 | println("\tCalculating RMSE with respect to reference solution\n")
57 |
58 | (rds.deltaT, RadialDiffusion.error(referenceSol)(solution))
59 |
60 | })
61 |
62 |
63 | val lossesSpace = bins.map(bL => {
64 |
65 | val rds = radialDiffusionSolver(bL, nT)
66 |
67 | println("Solving for delta L = "+rds.deltaL)
68 | val lShellVec = DenseVector.tabulate[Double](bL+1)(i =>
69 | if(i < bL) lShellLimits._1+(rds.deltaL*i)
70 | else lShellLimits._2).toArray.toSeq
71 |
72 | val timeVec = DenseVector.tabulate[Double](nT+1)(i =>
73 | if(i < nT) timeLimits._1+(rds.deltaT*i)
74 | else timeLimits._2).toArray.toSeq
75 |
76 | println("\tGenerating neural computation stack & computing solution")
77 |
78 | val solution = rds.solve(q, dll, loss)(initialPSD)
79 |
80 | val referenceSol = timeVec.map(t =>
81 | DenseVector(lShellVec.map(lS => referenceSolution(lS, t)).toArray))
82 |
83 | println("\tCalculating RMSE with respect to reference solution\n")
84 |
85 | (rds.deltaL, RadialDiffusion.error(referenceSol)(solution))
86 |
87 | })
88 |
89 |
90 |
91 | spline(lossesTime)
92 | title("Forward Solver Error")
93 | xAxisType(AxisType.logarithmic)
94 | xAxis("delta T")
95 | yAxis("RMSE")
96 |
97 |
98 | spline(lossesSpace)
99 | title("Forward Solver Error")
100 | xAxisType(AxisType.logarithmic)
101 | xAxis("delta L")
102 | yAxis("RMSE")
103 |
--------------------------------------------------------------------------------
/helios/scripts/helios_tl_exp.sc:
--------------------------------------------------------------------------------
1 | import $exec.helios.scripts.{
2 | timelagutils,
3 | timelag_inference_fixed => exp1,
4 | timelag_inference_const_v => exp2,
5 | timelag_inference_const_a => exp3,
6 | timelag_inference_softplus => exp4}
7 |
8 |
9 | val res_exp2 = exp2.main(
10 | d = 8, n = 4000, sliding_window = 20,
11 | noise = 0.5, noiserot = 0.001,
12 | alpha = 0.005, train_test_separate = true,
13 | num_neurons = Seq(40, 20),
14 | iterations = 150000, miniBatch = 1024,
15 | optimizer = tf.train.Adam(0.01),
16 | reg = 0.001, c = 1.0,
17 | prior_type = helios.learn.cdt_loss.JensenShannon,
18 | error_wt = 1.0, prior_wt = 0.75,
19 | mo_flag = true, prob_timelags = true,
20 | dist_type = "default")
21 |
22 | val res_exp2_sw = exp2.stage_wise(
23 | d = 8, n = 4000, sliding_window = 20,
24 | noise = 0.5, noiserot = 0.001,
25 | alpha = 0.005,
26 | num_neurons_i = Seq(40, 20),
27 | num_neurons_ii = Seq(30, 20),
28 | iterations = 150000, miniBatch = 1024,
29 | optimizer = tf.train.Adam(0.01),
30 | reg_i = 0.001, reg_ii = 0.0001, c = 1.0,
31 | prior_type = helios.learn.cdt_loss.JensenShannon,
32 | error_wt = 1.0, prior_wt = 0.75,
33 | mo_flag = true, prob_timelags = true,
34 | dist_type = "default")
35 |
36 |
37 | val res_exp3 = exp3.main(
38 | d = 8, n = 4000, sliding_window = 20,
39 | noise = 0.65, noiserot = 0.001,
40 | alpha = 0.005, train_test_separate = true,
41 | num_neurons = Seq(40, 25),
42 | iterations = 150000, miniBatch = 1024,
43 | optimizer = tf.train.Adam(0.01),
44 | reg = 0.001, c = 1.0,
45 | prior_type = "Kullback-Leibler",
46 | error_wt = 1.0, prior_wt = 0.75,
47 | mo_flag = true, prob_timelags = true,
48 | dist_type = "default")
49 |
50 |
51 | val res_exp1 = exp1.main(
52 | d = 8, n = 4000, sliding_window = 20,
53 | noise = 0.25, noiserot = 0.001,
54 | alpha = 0.0035, train_test_separate = true,
55 | num_neurons = Seq(15),
56 | iterations = 120000, miniBatch = 1024,
57 | optimizer = tf.train.Adam(0.001),
58 | reg = 0.0001, prior_wt = 0.8,
59 | prior_type = "Hellinger",
60 | temp = 0.75, error_wt = 1.0,
61 | mo_flag = true, prob_timelags = true)
62 |
63 |
64 | val res_exp4 = exp4.main(
65 | d = 8, n = 4000, sliding_window = 20,
66 | noise = 0.5, noiserot = 0.001,
67 | alpha = 0.0045, train_test_separate = true,
68 | num_neurons = Seq(20, 20),
69 | iterations = 25000, miniBatch = 1024,
70 | optimizer = tf.train.Adam(0.001),
71 | reg = 0.0001, c = 4.0,
72 | prior_type = "Kullback-Leibler",
73 | error_wt = 1.0, prior_wt = 0.75,
74 | mo_flag = true, prob_timelags = true,
75 | dist_type = "default")
76 |
77 |
78 | val res_exp2_32 = exp2.main(
79 | d = 32, n = 4000, sliding_window = 15,
80 | noise = 1.1, noiserot = 0.001,
81 | alpha = 0.007, train_test_separate = true,
82 | num_neurons = Seq(40, 30),
83 | iterations = 120000, miniBatch = 1024,
84 | optimizer = tf.train.Adam(0.01),
85 | reg = 0.01, prior_wt = 1.0,
86 | prior_type = "Hellinger",
87 | temp = 0.75, error_wt = 1.0,
88 | mo_flag = true, prob_timelags = true,
89 | dist_type = "default")
90 |
91 | val res_exp3_32 = exp3.main(
92 | d = 32, n = 4000, sliding_window = 15,
93 | noise = 1.1, noiserot = 0.001,
94 | alpha = 0.007, train_test_separate = true,
95 | num_neurons = Seq(40, 30),
96 | iterations = 120000, miniBatch = 1024,
97 | optimizer = tf.train.Adam(0.01),
98 | reg = 0.01, prior_wt = 1.0,
99 | prior_type = "Hellinger",
100 | temp = 0.75, error_wt = 1.0,
101 | mo_flag = true, dist_type = "default")
102 |
--------------------------------------------------------------------------------
/mag-core/src/main/java/io/github/mandar2812/PlasmaML/cdf/AttributeEntry.java:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.cdf;
2 |
3 | /**
4 | * Represents an entry in a global or variable attribute.
5 | *
6 | * @author Mark Taylor
7 | * @since 28 Jun 2013
8 | */
9 | public class AttributeEntry {
10 |
11 | private final DataType dataType_;
12 | private final Object rawValue_;
13 | private final int nitem_;
14 |
15 | /**
16 | * Constructor.
17 | *
18 | * @param dataType data type
19 | * @param rawValue array object storing original representation
20 | * of the object in the CDF (array of primitives or
21 | * Strings)
22 | * @param nitem number of items represented by the array
23 | */
24 | public AttributeEntry( DataType dataType, Object rawValue, int nitem ) {
25 | dataType_ = dataType;
26 | rawValue_ = rawValue;
27 | nitem_ = nitem;
28 | }
29 |
30 | /**
31 | * Returns the data type of this entry.
32 | *
33 | * @return data type
34 | */
35 | public DataType getDataType() {
36 | return dataType_;
37 | }
38 |
39 | /**
40 | * Returns the array object storing the original representation
41 | * of the object in the CDF. This is either an array of either
42 | * primitives or Strings.
43 | *
44 | * @return raw array value
45 | */
46 | public Object getRawValue() {
47 | return rawValue_;
48 | }
49 |
50 | /**
51 | * Returns the value of this entry as a convenient object.
52 | * If the item count is 1 it's the same as getItem(0),
53 | * and if the item count is >1 it's the same as the raw value.
54 | *
55 | * @return shaped entry value
56 | */
57 | public Object getShapedValue() {
58 | if ( nitem_ == 0 ) {
59 | return null;
60 | }
61 | else if ( nitem_ == 1 ) {
62 | return dataType_.getScalar( rawValue_, 0 );
63 | }
64 | else {
65 | return rawValue_;
66 | }
67 | }
68 |
69 | /**
70 | * Returns the number of items in this entry.
71 | *
72 | * @return item count
73 | */
74 | public int getItemCount() {
75 | return nitem_;
76 | }
77 |
78 | /**
79 | * Returns an object representing one of the items in this entry.
80 | * If the raw array is a primitive, the result is a wrapper object.
81 | *
82 | * @param itemIndex item index
83 | * @return value of item
84 | */
85 | public Object getItem( int itemIndex ) {
86 | return dataType_.getScalar( rawValue_,
87 | dataType_.getArrayIndex( itemIndex ) );
88 | }
89 |
90 | /**
91 | * Formats the value of this entry as a string.
92 | */
93 | @Override
94 | public String toString() {
95 | if ( rawValue_ == null || nitem_ == 0 ) {
96 | return "";
97 | }
98 | else {
99 | StringBuffer sbuf = new StringBuffer();
100 | for ( int i = 0; i < nitem_; i++ ) {
101 | if ( i > 0 ) {
102 | sbuf.append( ", " );
103 | }
104 | sbuf.append( dataType_
105 | .formatArrayValue( rawValue_,
106 | dataType_.getArrayIndex( i ) ) );
107 | }
108 | return sbuf.toString();
109 | }
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/helios/scripts/causal_time_lag_tuning3.sc:
--------------------------------------------------------------------------------
1 | import $exec.helios.scripts.{
2 | tl_const_v_tuning => tuning_exp2,
3 | tl_const_a_tuning => tuning_exp3,
4 | tl_softplus_tuning => tuning_exp4
5 | }
6 |
7 | import ammonite.ops.ImplicitWd._
8 |
9 | import org.platanios.tensorflow.api.learn.layers.Activation
10 |
11 | val getReLUAct2: Int => Int => Activation = (s: Int) =>
12 | (i: Int) => {
13 | if ((i - s) == 0) tf.learn.ReLU(s"Act_$i", 0.01f)
14 | else tf.learn.Sigmoid(s"Act_$i")
15 | }
16 |
17 | val exp_set2 = tuning_exp2.main(
18 | d = 10,
19 | size_training = 8000,
20 | size_test = 2000,
21 | sliding_window = 20,
22 | noise = 0.7,
23 | noiserot = 0.001,
24 | alpha = 0.02,
25 | train_test_separate = true,
26 | num_neurons = Seq(30, 25, 20),
27 | activation_func = getReLUAct2(1),
28 | iterations = 150000,
29 | iterations_tuning = 10000,
30 | miniBatch = 1024,
31 | optimizer = tf.train.AdaDelta(0.01),
32 | prior_type =
33 | Seq(helios.learn.cdt_loss.KullbackLeibler, helios.learn.cdt_loss.Hellinger),
34 | target_prob =
35 | Seq(helios.learn.cdt_loss.Boltzmann, helios.learn.cdt_loss.Uniform),
36 | dist_type = "default",
37 | num_samples = 20,
38 | hyper_optimizer = "gs",
39 | hyp_opt_iterations = Some(8),
40 | regularization_types = Seq("L2")
41 | )
42 |
43 | timelag.organize_results(exp_set2, home / 'tmp / 'results_3l_exp2)
44 | %%(
45 | 'tar,
46 | "-C",
47 | home / 'tmp,
48 | "-zcvf",
49 | home / 'tmp / "exp2_3l.tar.gz",
50 | "results_3l_exp2"
51 | )
52 |
53 | val exp_set3 = tuning_exp3.main(
54 | d = 10,
55 | size_training = 8000,
56 | size_test = 2000,
57 | sliding_window = 20,
58 | noise = 0.7,
59 | noiserot = 0.001,
60 | alpha = 0.02,
61 | train_test_separate = true,
62 | num_neurons = Seq(30, 25, 20),
63 | activation_func = getReLUAct2(1),
64 | iterations = 150000,
65 | iterations_tuning = 10000,
66 | miniBatch = 1024,
67 | optimizer = tf.train.AdaDelta(0.01),
68 | prior_type =
69 | Seq(helios.learn.cdt_loss.KullbackLeibler, helios.learn.cdt_loss.Hellinger),
70 | target_prob =
71 | Seq(helios.learn.cdt_loss.Boltzmann, helios.learn.cdt_loss.Uniform),
72 | dist_type = "default",
73 | num_samples = 20,
74 | hyper_optimizer = "gs",
75 | hyp_opt_iterations = Some(8),
76 | regularization_types = Seq("L2")
77 | )
78 |
79 | timelag.organize_results(exp_set3, home / 'tmp / 'results_3l_exp3)
80 | %%(
81 | 'tar,
82 | "-C",
83 | home / 'tmp,
84 | "-zcvf",
85 | home / 'tmp / "exp3_3l.tar.gz",
86 | "results_3l_exp3"
87 | )
88 |
89 | val exp_set4 = tuning_exp4.main(
90 | d = 10,
91 | size_training = 8000,
92 | size_test = 2000,
93 | sliding_window = 20,
94 | noise = 0.7,
95 | noiserot = 0.001,
96 | alpha = 0.02,
97 | train_test_separate = true,
98 | num_neurons = Seq(30, 25, 20),
99 | activation_func = getReLUAct2(1),
100 | iterations = 150000,
101 | iterations_tuning = 10000,
102 | miniBatch = 1024,
103 | optimizer = tf.train.AdaDelta(0.01),
104 | prior_type =
105 | Seq(helios.learn.cdt_loss.KullbackLeibler, helios.learn.cdt_loss.Hellinger),
106 | target_prob =
107 | Seq(helios.learn.cdt_loss.Boltzmann, helios.learn.cdt_loss.Uniform),
108 | dist_type = "default",
109 | num_samples = 20,
110 | hyper_optimizer = "gs",
111 | hyp_opt_iterations = Some(8),
112 | regularization_types = Seq("L2")
113 | )
114 |
115 | timelag.organize_results(exp_set4, home / 'tmp / 'results_3l_exp4)
116 | %%(
117 | 'tar,
118 | "-C",
119 | home / 'tmp,
120 | "-zcvf",
121 | home / 'tmp / "exp4_3l.tar.gz",
122 | "results_3l_exp4"
123 | )
124 |
--------------------------------------------------------------------------------
/mag-core/src/main/scala/io/github/mandar2812/PlasmaML/dynamics/diffusion/InverseMQPSDBasis.scala:
--------------------------------------------------------------------------------
1 | package io.github.mandar2812.PlasmaML.dynamics.diffusion
2 |
3 | import breeze.linalg.DenseVector
4 | import io.github.mandar2812.dynaml.analysis.RadialBasis
5 | import io.github.mandar2812.dynaml.pipes.{Basis, DataPipe}
6 | import io.github.mandar2812.dynaml.analysis.implicits._
7 | import spire.algebra.InnerProductSpace
8 |
9 | /**
10 | * Phase Space Density: Inverse Multi-Quadric Mesh-Based Radial Basis
11 | *
12 | * Implements a inverse multi-quadric radial basis expansion
13 | * for the phase space density with nodes placed on a regular
14 | * space time mesh.
15 | *
16 | * */
17 | class InverseMQPSDBasis(
18 | beta: Double
19 | )(lShellLimits: (Double, Double),
20 | nL: Int,
21 | timeLimits: (Double, Double),
22 | nT: Int,
23 | logScales: (Boolean, Boolean) = (false, false))
24 | extends PSDRadialBasis(lShellLimits, nL, timeLimits, nT, logScales) {
25 |
26 | require(beta > 0, "Beta parameter must be positive")
27 |
28 | val activation = RadialBasis.invMultiQuadric(beta)
29 |
30 | val field: InnerProductSpace[(Double, Double), Double] = innerProdTuple2
31 |
32 | override protected val f: ((Double, Double)) => DenseVector[Double] =
33 | (x: (Double, Double)) =>
34 | DenseVector(
35 | centers
36 | .zip(scales)
37 | .map(cs => {
38 | val d = field.minus(x, cs._1)
39 | val scaledDist = (d._1 / cs._2._1, d._2 / cs._2._2)
40 | val r = math.sqrt(field.dot(scaledDist, scaledDist))
41 |
42 | activation(r)
43 | })
44 | .toArray
45 | )
46 |
47 | override val f_l: ((Double, Double)) => DenseVector[Double] =
48 | (x: (Double, Double)) => {
49 | DenseVector(
50 | centers
51 | .zip(scales)
52 | .map(cs => {
53 | val (c, (theta_s, _)) = cs
54 |
55 | val d = field.minus(x, c)
56 |
57 | val scaledDist = d._1 / theta_s
58 |
59 | val g_l = 1d + scaledDist * scaledDist
60 |
61 | val invThetaS = 1 / theta_s
62 |
63 | -beta * invThetaS * math.abs(d._1) / g_l
64 | })
65 | .toArray
66 | ) *:*
67 | f(x)
68 | }
69 |
70 | override val f_ll: ((Double, Double)) => DenseVector[Double] =
71 | (x: (Double, Double)) => {
72 | DenseVector(
73 | centers
74 | .zip(scales)
75 | .map(cs => {
76 | val (c, (theta_s, _)) = cs
77 |
78 | val d = field.minus(x, c)
79 |
80 | val scaledDist = d._1 / theta_s
81 |
82 | val g_l = 1d + scaledDist * scaledDist
83 |
84 | val invThetaS = 1 / theta_s
85 |
86 | val sq = (s: Double) => s * s
87 |
88 | beta * invThetaS * (sq(d._1) * (0.5 * beta + 1) - g_l) / math
89 | .pow(g_l, 2 + 0.5 * beta)
90 | })
91 | .toArray
92 | ) *:*
93 | f(x)
94 | }
95 |
96 | override val f_t: ((Double, Double)) => DenseVector[Double] =
97 | (x: (Double, Double)) => {
98 |
99 | DenseVector(
100 | centers
101 | .zip(scales)
102 | .map(cs => {
103 | val (c, (_, theta_t)) = cs
104 |
105 | val d = field.minus(x, c)
106 |
107 | val scaledDist = d._2 / theta_t
108 |
109 | val g_t = 1d + scaledDist * scaledDist
110 |
111 | val invThetaT = 1 / theta_t
112 |
113 | -beta * invThetaT * math.abs(d._2) / g_t
114 | })
115 | .toArray
116 | ) *:*
117 | f(x)
118 | }
119 | }
120 |
--------------------------------------------------------------------------------
/helios/scripts/pdt_softplus_tuning.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.pipes._
2 | import _root_.io.github.mandar2812.dynaml.repl.Router.main
3 | import org.platanios.tensorflow.api.ops.training.optimizers.Optimizer
4 | import _root_.io.github.mandar2812.PlasmaML.helios
5 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
6 | import _root_.ammonite.ops._
7 | import org.platanios.tensorflow.api._
8 | import org.platanios.tensorflow.api.learn.layers.Activation
9 | import org.platanios.tensorflow.api.learn.layers.Layer
10 |
11 | import $file.run_model_tuning_pdt
12 |
13 | @main
14 | def main(
15 | d: Int = 3,
16 | confounding: Seq[Double] = Seq(0d, 0.25, 0.5, 0.75),
17 | size_training: Int = 100,
18 | size_test: Int = 50,
19 | sliding_window: Int = 15,
20 | noise: Double = 0.5,
21 | noiserot: Double = 0.1,
22 | alpha: Double = 0.0,
23 | train_test_separate: Boolean = false,
24 | num_neurons: Seq[Int] = Seq(40),
25 | activation_func: Int => Layer[Output[Double], Output[Double]]
26 | = timelag.utils.getReLUAct[Double](1, _),
27 | iterations: Int = 150000,
28 | iterations_tuning: Int = 20000,
29 | pdt_iterations: Int = 2,
30 | pdt_iterations_tuning: Int = 4,
31 | miniBatch: Int = 32,
32 | optimizer: Optimizer = tf.train.AdaDelta(0.01f),
33 | sum_dir_prefix: String = "softplus",
34 | summaries_top_dir: Path = home/'tmp,
35 | num_samples: Int = 20,
36 | hyper_optimizer: String = "gs",
37 | hyp_opt_iterations: Option[Int] = Some(5),
38 | epochFlag: Boolean = false,
39 | regularization_types: Seq[String] = Seq("L2"),
40 | checkpointing_freq: Int = 4)
41 | : Seq[timelag.ExperimentResult[Double, Double, timelag.TunedModelRun[Double, Double]]] = {
42 |
43 | //Output computation
44 | val beta = 100f
45 |
46 | val compute_v = DataPipe[Tensor[Double], Float]((v: Tensor[Double]) => v.square.mean().scalar.asInstanceOf[Float]*beta/d + 40f)
47 |
48 | val compute_output: DataPipe[Tensor[Double], (Float, Float)] = DataPipe(
49 | (x: Tensor[Double]) => {
50 |
51 | val out = compute_v(x)
52 |
53 | (math.log(1 + math.exp(out/20.0)).toFloat, out + scala.util.Random.nextGaussian().toFloat)
54 | })
55 |
56 | val experiment_results = run_model_tuning_pdt(
57 | compute_output,
58 | d, confounding, size_training, size_test, sliding_window, noise, noiserot,
59 | alpha, train_test_separate, num_neurons,
60 | activation_func, iterations, iterations_tuning,
61 | pdt_iterations, pdt_iterations_tuning,
62 | miniBatch, optimizer, sum_dir_prefix,
63 | summaries_top_dir, num_samples,
64 | hyper_optimizer, hyp_opt_iterations, epochFlag,
65 | regularization_types, checkpointing_freq
66 | )
67 |
68 | experiment_results.map(experiment_result =>
69 | experiment_result.copy[Double, Double, timelag.TunedModelRun[Double, Double]](
70 | config = experiment_result.config.copy[Double](output_mapping = Some(compute_v))
71 | ))
72 | }
--------------------------------------------------------------------------------
/helios/scripts/pdt_const_lag_tuning.sc:
--------------------------------------------------------------------------------
1 | import _root_.io.github.mandar2812.dynaml.pipes._
2 | import _root_.io.github.mandar2812.dynaml.repl.Router.main
3 | import org.platanios.tensorflow.api.ops.training.optimizers.Optimizer
4 | import _root_.io.github.mandar2812.PlasmaML.helios
5 | import _root_.io.github.mandar2812.PlasmaML.helios.core.timelag
6 | import _root_.ammonite.ops._
7 | import org.platanios.tensorflow.api._
8 | import org.platanios.tensorflow.api.learn.layers.Activation
9 | import org.platanios.tensorflow.api.learn.layers.Layer
10 |
11 | import $file.run_model_tuning_pdt
12 |
13 | @main
14 | def main(
15 | fixed_lag: Int = 5,
16 | d: Int = 10,
17 | confounding: Seq[Double] = Seq(0d, 0.25, 0.5, 0.75),
18 | size_training: Int = 100,
19 | size_test: Int = 50,
20 | sliding_window: Int = 15,
21 | noise: Double = 0.5,
22 | noiserot: Double = 0.1,
23 | alpha: Double = 0.0,
24 | train_test_separate: Boolean = false,
25 | num_neurons: Seq[Int] = Seq(40),
26 | activation_func: Int => Layer[Output[Double], Output[Double]]
27 | = timelag.utils.getReLUAct[Double](1, _),
28 | iterations: Int = 150000,
29 | iterations_tuning: Int = 20000,
30 | pdt_iterations: Int = 2,
31 | pdt_iterations_tuning: Int = 4,
32 | miniBatch: Int = 32,
33 | optimizer: Optimizer = tf.train.AdaDelta(0.01f),
34 | sum_dir_prefix: String = "const_lag",
35 | summaries_top_dir: Path = home/'tmp,
36 | num_samples: Int = 20,
37 | hyper_optimizer: String = "gs",
38 | hyp_opt_iterations: Option[Int] = Some(5),
39 | epochFlag: Boolean = false,
40 | regularization_types: Seq[String] = Seq("L2"),
41 | checkpointing_freq: Int = 4)
42 | : Seq[timelag.ExperimentResult[Double, Double, timelag.TunedModelRun[Double, Double]]] = {
43 |
44 |
45 | val beta = 100f
46 | //Output computation
47 | val compute_v = DataPipe[Tensor[Double], Float]((v: Tensor[Double]) => v.square.mean().scalar.asInstanceOf[Float]*beta/d)
48 |
49 | val compute_output: DataPipe[Tensor[Double], (Float, Float)] = DataPipe(
50 | (x: Tensor[Double]) => {
51 |
52 | val out = compute_v(x)
53 |
54 | (fixed_lag.toFloat, out + scala.util.Random.nextGaussian().toFloat)
55 | })
56 |
57 |
58 | val experiment_results = run_model_tuning_pdt(
59 | compute_output,
60 | d, confounding, size_training, size_test,
61 | sliding_window, noise, noiserot,
62 | alpha, train_test_separate, num_neurons,
63 | activation_func, iterations, iterations_tuning,
64 | pdt_iterations, pdt_iterations_tuning,
65 | miniBatch, optimizer, sum_dir_prefix,
66 | summaries_top_dir, num_samples,
67 | hyper_optimizer, hyp_opt_iterations, epochFlag,
68 | regularization_types, checkpointing_freq
69 | )
70 |
71 | experiment_results.map(experiment_result =>
72 | experiment_result.copy[Double, Double, timelag.TunedModelRun[Double, Double]](
73 | config = experiment_result.config.copy[Double](output_mapping = Some(compute_v))
74 | ))
75 | }
--------------------------------------------------------------------------------
/mag-core/scripts/visualiseSensitivity.R:
--------------------------------------------------------------------------------
1 | #/usr/bin/Rscript
2 | library(ggplot2)
3 | library(latex2exp)
4 |
5 | args <- commandArgs(trailingOnly = TRUE)
6 | direc <- args[1]
7 | lossFlag <- args[2]
8 | setwd(direc)
9 |
10 | palette2 <- c("firebrick3", "#000000")
11 | palette3 <- c("#CC0C0099", "#5C88DA99")
12 |
13 | psd_df <- read.csv("solution.csv")
14 | s_df <- read.csv("sensitivity.csv")
15 | s_df <- subset(s_df, parameter != "gamma")
16 |
17 | s_df_q <- subset(s_df, quantity == "Q")
18 | s_df_b <- subset(s_df, parameter == "b")
19 | s_df_rest <- subset(s_df, parameter != "b" & quantity != "Q")
20 |
21 | s_df$parameter <- factor(as.character(s_df$parameter), labels = c("alpha", "b", "beta"))
22 | s_df_q$parameter <- factor(as.character(s_df_q$parameter), labels = c("alpha", "b", "beta"))
23 | s_df_rest$parameter <- factor(as.character(s_df_rest$parameter), labels = c("alpha", "beta"))
24 |
25 |
26 | s_df$quantity <- factor(
27 | s_df$quantity,
28 | levels = c("Q", "dll", "lambda"),
29 | labels = unname(TeX(c('$q(L^*, t)$', '$\\kappa(L^*, t)$', '$\\lambda(L^*, t)$'))))
30 |
31 | s_df_b$quantity <- factor(
32 | s_df_b$quantity,
33 | levels = c("Q", "dll", "lambda"),
34 | labels = unname(TeX(c('$q(L^*, t)$', '$\\kappa(L^*, t)$', '$\\lambda(L^*, t)$'))))
35 |
36 | s_df_rest$quantity <- factor(
37 | s_df_rest$quantity,
38 | levels = c("dll", "lambda"),
39 | labels = unname(TeX(c('$\\kappa(L^*, t)$', '$\\lambda(L^*, t)$')))
40 | )
41 |
42 | #c("kappa(L^*, t)", "lambda(L^*, t)", "Q(L^*, t)")
43 |
44 | ggplot(psd_df, aes(x = t, y = l)) +
45 | geom_raster(aes(fill = psd)) +
46 | scale_fill_viridis_c(name = unname(TeX('$f(L^{*}, t)$'))) +
47 | theme_gray(base_size = 20) +
48 | ylab(TeX('$L^{*}$')) +
49 | xlab(TeX('$t$')) +
50 | theme(
51 | #legend.position="top",
52 | #legend.direction="vertical")#,
53 | legend.text=element_text(size=14))
54 |
55 | ggsave("psd.pdf", scale=1.5)
56 |
57 |
58 |
59 | ggplot(s_df, aes(x = t, y = l)) +
60 | geom_raster(aes(fill = value)) +
61 | facet_grid(quantity ~ parameter, labeller = label_parsed) +
62 | scale_fill_viridis_c(name = "") +
63 | theme_gray(base_size = 20) +
64 | ylab(TeX('$L^{*}$')) +
65 | xlab(TeX('$t$')) +
66 | theme(axis.text.x = element_text(angle = 90), legend.text=element_text(size=14))
67 |
68 | ggsave("sensitivity.pdf", scale = 2.0)
69 |
70 | ggplot(s_df_q, aes(x = t, y = l)) +
71 | geom_raster(aes(fill = value)) +
72 | facet_grid(parameter~., labeller = label_parsed) +
73 | scale_fill_viridis_c(name = TeX('$s(L^{*}, t)$')) +
74 | theme_gray(base_size = 20) +
75 | ylab(TeX('$L^{*}$')) +
76 | xlab(TeX('$t$')) +
77 | theme(
78 | #legend.position="top",
79 | #legend.direction="vertical")#,
80 | legend.text=element_text(size=14))
81 |
82 | ggsave("sensitivity_Q.pdf", scale=1.5)
83 |
84 | ggplot(s_df_b, aes(x = t, y = l)) +
85 | geom_raster(aes(fill = value)) +
86 | facet_grid(quantity~., labeller = label_parsed) +
87 | scale_fill_viridis_c(name = TeX('$s(L^{*}, t)$')) +
88 | theme_gray(base_size = 20) +
89 | ylab(TeX('$L^{*}$')) +
90 | xlab(TeX('$t$')) +
91 | theme(
92 | #legend.position="top",
93 | #legend.direction="vertical")#,
94 | legend.text=element_text(size=14))
95 |
96 | ggsave("sensitivity_b.pdf", scale=1.5)
97 |
98 | ggplot(s_df_rest, aes(x = t, y = l)) +
99 | geom_raster(aes(fill = value)) +
100 | facet_grid(quantity ~ parameter, labeller = label_parsed) +
101 | scale_fill_viridis_c(name = TeX('$s(L^{*}, t)$')) +
102 | theme_gray(base_size = 20) +
103 | ylab(TeX('$L^{*}$')) +
104 | xlab(TeX('$t$')) +
105 | theme(
106 | #legend.position="top",
107 | #legend.direction="vertical")#,
108 | legend.text=element_text(size=14))
109 |
110 | ggsave("sensitivity_rest.pdf", scale=1.5)
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # PlasmaML
2 |
3 | [](https://travis-ci.org/transcendent-ai-labs/PlasmaML)
4 |
5 | Machine Learning tools for Space Weather and Plasma Physics
6 | ---------------------------
7 | > 
8 | >
9 | > courtesy [NASA](www.nasa.gov)
10 |
11 | *PlasmaML* is a collection of data analysis and machine learning tools in the domain of space physics, more specifically in modelling of space plasmas & space weather prediction.
12 |
13 | This is a multi-language project where the primary modelling is done in *Scala* while *R* is heavily leveraged for generating visualizations. The project depends on the [DynaML](https://github.com/mandar2812/DynaML) scala machine learning library and uses model and optimization implementations in it as a starting point for extensive experiments in space physics simulations and space weather prediction.
14 |
15 | ## Getting Started
16 |
17 | *PlasmaML* is managed using the Simple Build Tool (sbt).
18 |
19 | ### Installation
20 |
21 | #### Requirements
22 |
23 | 1. Java Development Kit 8.
24 | 2. [Scala](scala-lang.org)
25 | 3. [sbt](http://www.scala-sbt.org/)
26 | 4. [R](https://www.r-project.org/) with the following packages:
27 |
28 | * `ggplot2`
29 | * `reshape2`
30 | * `latex2exp`
31 | * `plyr`
32 | * `gridExtra`
33 | * `reshape2`
34 | * `directlabels`
35 |
36 |
37 | #### Steps
38 |
39 | After cloning the project, PlasmaML can be installed directly from the shell or
40 | by first entering the sbt shell and building the source.
41 |
42 | **From the shell**
43 |
44 | From the root directory `PlasmaML` run the build script (with configurable parameters).
45 |
46 | ```bash
47 | ./build.sh