├── tools ├── check-git-modified ├── read-nbs ├── run-after-git-clone ├── fastai-nbstripout ├── trust-origin-git-config ├── export_import.ipynb └── .ipynb_checkpoints │ └── export_import-checkpoint.ipynb ├── Makefile ├── Package.swift ├── Sources ├── SwiftAI │ ├── Matmul.swift │ ├── FullyConnected.swift │ ├── WhySqrt5.swift │ ├── Cuda.swift │ ├── HeterogeneousDictionary.swift │ ├── MinibatchTraining.swift │ ├── MixupLs.swift │ ├── EarlyStopping.swift │ ├── Imagenette.swift │ ├── LoadData.swift │ ├── Anneal.swift │ ├── Batchnorm.swift │ ├── DataBlock.swift │ ├── Callbacks.swift │ ├── FastaiLayers.swift │ └── Optimizer.swift └── run │ └── main.swift ├── Dockerfile ├── .gitconfig ├── .gitignore ├── .github └── workflows │ └── main.yml ├── README.md ├── LICENSE └── nbs ├── 08a_heterogeneous_dictionary.ipynb └── 02a_why_sqrt5.ipynb /tools/check-git-modified: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | 6 | path = sys.argv[1] 7 | res = os.popen(f'git status -uno -s {path}').read() 8 | if len(res) > 0: 9 | print(f"{path} has been modified") 10 | exit(1) 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | build: 4 | docker build -t swiftai . 5 | 6 | shell: build 7 | docker run --privileged --user $(id -u):$(id -g) -p 127.0.0.1:8888:8888 -v $(PWD):/root/swiftai -it swiftai /bin/bash 8 | 9 | jupyter: build 10 | docker run --privileged --user $(id -u):$(id -g) -p 127.0.0.1:8888:8888 -v $(PWD):/root/swiftai swiftai 11 | 12 | strip: 13 | ./tools/fastai-nbstripout -d nbs/* 14 | 15 | convert-nbs-to-srcs: 16 | ./tools/check-git-modified ./Sources 17 | jupyter nbconvert --execute tools/export_import.ipynb 18 | 19 | sync-nbs-to-srcs: convert-nbs-to-srcs strip 20 | -------------------------------------------------------------------------------- /tools/read-nbs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import nbformat, os 4 | from pathlib import Path 5 | 6 | def read_nbs(path): 7 | "Check all notebooks in `path` (and subfolders) can be opened" 8 | path,nb_files = Path(path),[] 9 | for p,d,f in os.walk(path): nb_files += [Path(p)/f_ for f_ in f if f_.endswith('.ipynb')] 10 | for nb in nb_files: 11 | try: 12 | with open(nb, 'r') as f: _ = nbformat.reads(f.read(), as_version=4) 13 | except Exception as e: 14 | print(f"{nb} is corrupted and can't be opened.") 15 | raise e 16 | 17 | read_nbs('nbs') 18 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | import PackageDescription 3 | 4 | let package = Package( 5 | name: "SwiftAI", 6 | platforms: [ 7 | .macOS(.v10_13), 8 | ], 9 | products: [ 10 | .library(name: "SwiftAI", targets: ["SwiftAI"]), 11 | ], 12 | dependencies: [ 13 | .package(url: "https://github.com/saeta/Just", from: "0.7.3"), 14 | .package(url: "https://github.com/mxcl/Path.swift", from: "0.16.3"), 15 | ], 16 | targets: [ 17 | .target( name: "SwiftAI", dependencies: ["Just", "Path"]), 18 | .target( name: "run", dependencies: ["SwiftAI"]), 19 | ] 20 | ) 21 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Matmul.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 01_matmul.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | import TensorFlow 14 | 15 | //cell90 16 | public extension StringTensor { 17 | // Read a file into a Tensor. 18 | init(readFile filename: String) { 19 | self.init(readFile: StringTensor(filename)) 20 | } 21 | init(readFile filename: StringTensor) { 22 | self = _Raw.readFile(filename: filename) 23 | } 24 | 25 | // Decode a StringTensor holding a JPEG file into a Tensor. 26 | func decodeJpeg(channels: Int = 0) -> Tensor { 27 | return _Raw.decodeJpeg(contents: self, channels: Int64(channels), dctMethod: "") 28 | } 29 | } 30 | 31 | -------------------------------------------------------------------------------- /Sources/SwiftAI/FullyConnected.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 02_fully_connected.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | 15 | //cell6 16 | public typealias TF=Tensor 17 | 18 | //cell8 19 | public func normalize(_ x:TF, mean:TF, std:TF) -> TF { 20 | return (x-mean)/std 21 | } 22 | 23 | //cell14 24 | public func testNearZero(_ a: TF, tolerance: Float = 1e-3) { 25 | assert((abs(a) .< tolerance).all(), "Near zero: \(a)") 26 | } 27 | 28 | public func testSame(_ a: TF, _ b: TF) { 29 | // Check shapes match so broadcasting doesn't hide shape errors. 30 | assert(a.shape == b.shape) 31 | testNearZero(a-b) 32 | } 33 | 34 | //cell39 35 | public func mse(_ out: TF, _ targ: TF) -> TF { 36 | return (out.squeezingShape(at: -1) - targ).squared().mean() 37 | } 38 | -------------------------------------------------------------------------------- /Sources/run/main.swift: -------------------------------------------------------------------------------- 1 | import SwiftAI 2 | import TensorFlow 3 | 4 | let path = downloadImagenette() 5 | let il = ItemList(fromFolder: path, extensions: ["jpeg", "jpg"]) 6 | let sd = SplitData(il) {grandParentSplitter(fName: $0, valid: "val")} 7 | var procLabel = CategoryProcessor() 8 | let sld = makeLabeledData(sd, fromFunc: parentLabeler, procLabel: &procLabel) 9 | let rawData = sld.toDataBunch(itemToTensor: pathsToTensor, labelToTensor: intsToTensor, bs: 128) 10 | let data = transformData(rawData) { openAndResize(fname: $0, size: 128) } 11 | let batch = data.train.oneBatch()! 12 | print("x / y batch shape:", batch.xb.shape,batch.yb.shape) 13 | 14 | func modelInit() -> XResNet { return xresnet18(cOut: 10) } 15 | let optFunc: (XResNet) -> StatefulOptimizer = adamOpt(lr: 1e-3, mom: 0.9, beta: 0.99, wd: 1e-2, eps: 1e-4) 16 | let learner = Learner(data: data, lossFunc: crossEntropy, optFunc: optFunc, modelInit: modelInit) 17 | let recorder = learner.makeDefaultDelegates(metrics: [accuracy]) 18 | learner.addDelegate(learner.makeNormalize(mean: imagenetStats.mean, std: imagenetStats.std)) 19 | learner.addOneCycleDelegates(1e-3, pctStart: 0.5) 20 | try! learner.fit(1) 21 | 22 | -------------------------------------------------------------------------------- /tools/run-after-git-clone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # if you're a fastai developer please make sure you do this: 4 | # 5 | # git clone https://github.com/fastai/fastai_docs 6 | # cd fastai_docs 7 | # tools/run-after-git-clone 8 | # 9 | 10 | import sys 11 | if sys.hexversion < 0x03060000: sys.exit("!!! Please re-run this script with python-3.6 or higher") 12 | 13 | import subprocess, os 14 | from pathlib import Path 15 | 16 | def run_script(script): 17 | 18 | cmd = f"{sys.executable} {script}" 19 | #print(f"Executing: {cmd}") 20 | result = subprocess.run(cmd.split(), shell=False, check=False, 21 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 22 | 23 | if result.returncode != 0: print(f"Failed to execute: {cmd}") 24 | if result.stdout: print(f"{result.stdout.decode('utf-8')}") 25 | if result.stderr: print(f"Error: {result.stderr.decode('utf-8')}") 26 | 27 | # make sure we are under the root of the project 28 | cur_dir = Path(".").resolve().name 29 | if (cur_dir == "tools"): os.chdir("..") 30 | 31 | path = Path("tools") 32 | 33 | # facilitate trusting of the repo-wide .gitconfig 34 | run_script(path/"trust-origin-git-config") 35 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # README 2 | # If you ever intend to add anything via the ADD verb, please remove the wildcard from .dockerignore 3 | 4 | 5 | FROM ubuntu:18.04 6 | 7 | # Update base 8 | RUN apt update 9 | RUN apt upgrade -y 10 | 11 | # Install deps and useful utils 12 | RUN apt install -y apt-utils clang curl git libblocksruntime-dev libpython-dev libpython3.6 libxml2 python3 python3-pip 13 | 14 | # Install S4TF 15 | WORKDIR /root 16 | RUN curl https://storage.googleapis.com/swift-tensorflow-artifacts/nightlies/latest/swift-tensorflow-DEVELOPMENT-ubuntu18.04.tar.gz > swift.tar.gz 17 | RUN tar -xf swift.tar.gz 18 | ENV PATH="/root/usr/bin:${PATH}" 19 | 20 | # Install swift-jupyter and register 21 | RUN git clone https://github.com/google/swift-jupyter.git 22 | WORKDIR /root/swift-jupyter 23 | RUN pip3 install jupyter 24 | RUN python3 register.py --sys-prefix --swift-toolchain /root 25 | 26 | # Install fastai since nbs use it 27 | RUN pip3 install fastai 28 | 29 | # Set default workdir to swiftai 30 | WORKDIR /root/swiftai 31 | 32 | # Good practice to expose ports for future automated tooling 33 | EXPOSE 8888 34 | 35 | # Bind to all available interfaces 36 | CMD jupyter notebook --ip=0.0.0.0 --allow-root 37 | -------------------------------------------------------------------------------- /.gitconfig: -------------------------------------------------------------------------------- 1 | # You need to enable this configuration once after checking out the repo 2 | # for the first time (assuming you checked it out into the swiftai/ dir): 3 | # 4 | # cd swiftai 5 | # git config --local include.path ../.gitconfig 6 | # 7 | # If you need to disable this instrumentation do: 8 | # 9 | # git config --local --unset include.path 10 | # 11 | # You can always check .git/config to see whether a ../.gitconfig 12 | # [include] entry is there or not. 13 | # 14 | # If tools/fastai-nbstripout is modified to produce a different output, 15 | # manually rerun all the notebooks under git: 16 | # 17 | # tools/fastai-nbstripout -d nbs/*/*ipynb 18 | # 19 | # # disable the strip out filter to get git to see changes 20 | # git config --local --unset include.path 21 | # git commit -a 22 | # git push 23 | # # restore the filter 24 | # git config --local include.path ../.gitconfig 25 | # 26 | 27 | # code notebooks strip out 28 | [filter "fastai-nbstripout-code"] 29 | clean = tools/fastai-nbstripout 30 | smudge = cat 31 | required = true 32 | [diff "ipynb-code"] 33 | textconv = tools/fastai-nbstripout -t 34 | 35 | # docs notebooks strip out 36 | [filter "fastai-nbstripout-docs"] 37 | clean = tools/fastai-nbstripout -d 38 | smudge = cat 39 | required = true 40 | [diff "ipynb-docs"] 41 | textconv = tools/fastai-nbstripout -dt 42 | -------------------------------------------------------------------------------- /Sources/SwiftAI/WhySqrt5.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 02a_why_sqrt5.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Foundation 13 | import TensorFlow 14 | import Path 15 | 16 | //cell17 17 | func leakyRelu( 18 | _ x: Tensor, 19 | negativeSlope: Double = 0.0 20 | ) -> Tensor { 21 | return max(0, x) + T(negativeSlope) * min(0, x) 22 | } 23 | 24 | //cell28 25 | extension Tensor where Scalar: TensorFlowFloatingPoint { 26 | init(kaimingUniform shape: TensorShape, negativeSlope: Double = 1.0) { 27 | // Assumes Leaky ReLU nonlinearity 28 | let gain = Scalar.init(TensorFlow.sqrt(2.0 / (1.0 + TensorFlow.pow(negativeSlope, 2)))) 29 | let spatialDimCount = shape.count - 2 30 | let receptiveField = shape[0.. AddChannel { return AddChannel() } 30 | } 31 | 32 | //cell10 33 | public struct CnnModel: Layer { 34 | public var convs: [FAConv2D] 35 | public var pool = FAGlobalAvgPool2D() 36 | public var linear: FADense 37 | 38 | public init(channelIn: Int, nOut: Int, filters: [Int]){ 39 | let allFilters = [channelIn] + filters 40 | convs = Array(0..(filters.last!, nOut) 44 | } 45 | 46 | @differentiable 47 | public func callAsFunction(_ input: TF) -> TF { 48 | return linear(pool(convs(input))) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | swift-install/ 3 | *.swp 4 | *~ 5 | Package.resolved 6 | 7 | # Xcode 8 | 9 | ## Build generated 10 | build/ 11 | DerivedData/ 12 | 13 | ## Various settings 14 | *.pbxuser 15 | !default.pbxuser 16 | *.mode1v3 17 | !default.mode1v3 18 | *.mode2v3 19 | !default.mode2v3 20 | *.perspectivev3 21 | !default.perspectivev3 22 | xcuserdata/ 23 | 24 | ## Other 25 | *.moved-aside 26 | *.xccheckout 27 | *.xcscmblueprint 28 | 29 | ## Obj-C/Swift specific 30 | *.hmap 31 | *.ipa 32 | *.dSYM.zip 33 | *.dSYM 34 | 35 | ## Playgrounds 36 | timeline.xctimeline 37 | playground.xcworkspace 38 | 39 | # Swift Package Manager 40 | # 41 | # Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. 42 | # Packages/ 43 | # Package.pins 44 | # Package.resolved 45 | .build/ 46 | 47 | # CocoaPods 48 | # 49 | # We recommend against adding the Pods directory to your .gitignore. However 50 | # you should judge for yourself, the pros and cons are mentioned at: 51 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 52 | # 53 | # Pods/ 54 | 55 | # Carthage 56 | # 57 | # Add this line if you want to avoid checking in source code from Carthage dependencies. 58 | # Carthage/Checkouts 59 | 60 | Carthage/Build 61 | 62 | # fastlane 63 | # 64 | # It is recommended to not store the screenshots in the git repo. Instead, use fastlane to re-generate the 65 | # screenshots whenever they are needed. 66 | # For more information about the recommended setup visit: 67 | # https://docs.fastlane.tools/best-practices/source-control/#source-control 68 | 69 | fastlane/report.xml 70 | fastlane/Preview.html 71 | fastlane/screenshots/**/*.png 72 | fastlane/test_output 73 | -------------------------------------------------------------------------------- /Sources/SwiftAI/HeterogeneousDictionary.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 08a_heterogeneous_dictionary.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | 14 | //cell3 15 | public protocol HetDictKey { 16 | associatedtype ValueType 17 | static var defaultValue: ValueType { get } 18 | } 19 | 20 | //cell4 21 | 22 | public struct HeterogeneousDictionary { 23 | private var underlying: [ObjectIdentifier : Any] = [:] 24 | 25 | public init() {} 26 | public init(_ key: T.Type, _ value: T.ValueType) { 27 | self.underlying = [ObjectIdentifier(key): value] 28 | } 29 | public init(_ key1: T1.Type, _ value1: T1.ValueType, _ key2: T2.Type, _ value2: T2.ValueType) { 30 | self.underlying = [ObjectIdentifier(key1): value1, ObjectIdentifier(key2): value2] 31 | } 32 | 33 | public subscript(key: T.Type) -> T.ValueType { 34 | get { return underlying[ObjectIdentifier(key), default: T.defaultValue] as! T.ValueType } 35 | set { underlying[ObjectIdentifier(key)] = newValue as Any } 36 | } 37 | 38 | public mutating func merge(_ other: HeterogeneousDictionary, 39 | uniquingKeysWith combine: (Any, Any) throws -> Any) rethrows { 40 | try self.underlying.merge(other.underlying, uniquingKeysWith: combine) 41 | } 42 | } 43 | 44 | 45 | //cell5 46 | // Common keys 47 | public struct LearningRate: HetDictKey { 48 | public static var defaultValue: Float = 0.4 49 | } 50 | -------------------------------------------------------------------------------- /Sources/SwiftAI/MinibatchTraining.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 03_minibatch_training.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | 15 | //cell5 16 | public typealias TI = Tensor 17 | 18 | //cell43 19 | public func accuracy(_ output: TF, _ target: TI) -> TF{ 20 | let corrects = TF(output.argmax(squeezingAxis: 1) .== target) 21 | return corrects.mean() 22 | } 23 | 24 | //cell64 25 | public func batchedRanges(start:Int, end:Int, bs:Int) -> UnfoldSequence,Int> 26 | { 27 | return sequence(state: start) { (batchStart) -> Range? in 28 | let remaining = end - batchStart 29 | guard remaining > 0 else { return nil} 30 | let currentBs = min(bs,remaining) 31 | let batchEnd = batchStart.advanced(by: currentBs) 32 | defer { batchStart = batchEnd } 33 | return batchStart ..< batchEnd 34 | } 35 | } 36 | 37 | //cell68 38 | public struct DataBatch: TensorGroup { 39 | public var xb: Inputs 40 | public var yb: Labels 41 | 42 | public init(xb: Inputs, yb: Labels){ (self.xb,self.yb) = (xb,yb) } 43 | 44 | public var _tensorHandles: [_AnyTensorHandle] { 45 | xb._tensorHandles + yb._tensorHandles 46 | } 47 | 48 | public init(_handles: C) where C.Element: _AnyTensorHandle { 49 | let xStart = _handles.startIndex 50 | let xEnd = _handles.index( 51 | xStart, offsetBy: Int(Inputs._tensorHandleCount)) 52 | self.xb = Inputs.init(_handles: _handles[xStart.. TF { 60 | return softmaxCrossEntropy(logits: logits, labels: labels) 61 | } 62 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | 13 | - name: Install apt dependencies 14 | run: | 15 | sudo apt install -y apt-utils clang curl git libblocksruntime-dev libxml2 16 | 17 | - uses: actions/setup-python@v1 18 | with: 19 | python-version: '3.6' 20 | architecture: 'x64' 21 | 22 | - name: Install pip dependencies 23 | run: | 24 | python --version 25 | pip install nbformat jupyter 26 | 27 | - name: Detect unstripped out notebook commits 28 | run: | 29 | echo "Trying to load all notebooks" 30 | tools/read-nbs 31 | echo "Check we are starting with clean git checkout" 32 | if [ -n "$(git status -uno -s)" ]; then echo "git status is not clean"; false; fi 33 | echo "Trying to strip out notebooks" 34 | make strip 35 | echo "Check that strip out was unnecessary" 36 | git status -s # display the status to see which nbs need cleaning up 37 | if [ -n "$(git status -uno -s)" ]; then echo -e "!!! Detected unstripped out notebooks\n!!!Remember to run tools/run-after-git-clone"; false; fi 38 | 39 | - name: Install Swift Tensorflow 40 | run: | 41 | curl https://storage.googleapis.com/swift-tensorflow-artifacts/nightlies/latest/swift-tensorflow-DEVELOPMENT-ubuntu18.04.tar.gz > swift.tar.gz 42 | mkdir swift 43 | tar -C swift -xf swift.tar.gz 44 | 45 | - name: Install Jupyter Swift Hooks 46 | run: | 47 | export PATH=$(pwd)/swift/usr/bin:${PATH} 48 | git clone https://github.com/google/swift-jupyter.git 49 | cd swift-jupyter 50 | python register.py --sys-prefix --swift-toolchain $(pwd)/../swift 51 | 52 | - name: Diff nbs/module 53 | run: | 54 | export PATH=$(pwd)/swift/usr/bin:${PATH} 55 | make sync-nbs-to-srcs 56 | git status -s # display the status to see what changed 57 | if [ -n "$(git status -uno -s)" ]; then echo -e "!!! Detected changes in notebooks or sources\n!!!Remember to run export_import.ipynb"; false; fi 58 | make strip 59 | 60 | - name: Build SwiftAI 61 | run: | 62 | export PATH=$(pwd)/swift/usr/bin:${PATH} 63 | swift build 64 | #swift test ## enable when tests are present 65 | -------------------------------------------------------------------------------- /Sources/SwiftAI/MixupLs.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 10_mixup_ls.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | import TensorFlow 14 | 15 | //cell12 16 | extension RandomDistribution { 17 | // Returns a batch of samples. 18 | func next( 19 | _ count: Int, using generator: inout G 20 | ) -> [Sample] { 21 | var result: [Sample] = [] 22 | for _ in 0.. [Sample] { 30 | return next(count, using: &ThreefryRandomNumberGenerator.global) 31 | } 32 | } 33 | 34 | //cell14 35 | extension Learner { 36 | public class MixupDelegate: Delegate { 37 | private var distribution: BetaDistribution 38 | 39 | public init(alpha: Float = 0.4){ 40 | distribution = BetaDistribution(alpha: alpha, beta: alpha) 41 | } 42 | 43 | override public func batchWillStart(learner: Learner) { 44 | if let xb = learner.currentInput { 45 | if let yb = learner.currentTarget as? Tensor{ 46 | var lambda = Tensor(distribution.next(Int(yb.shape[0]))) 47 | lambda = max(lambda, 1-lambda) 48 | let shuffle = _Raw.randomShuffle(value: Tensor(0.. MixupDelegate { 60 | return MixupDelegate(alpha: alpha) 61 | } 62 | } 63 | 64 | //cell26 65 | @differentiable(wrt: out) 66 | public func labelSmoothingCrossEntropy(_ out: TF, _ targ: TI, ε: Float = 0.1) -> TF { 67 | let c = out.shape[1] 68 | let loss = softmaxCrossEntropy(logits: out, labels: targ) 69 | let logPreds = logSoftmax(out) 70 | return (1-ε) * loss - (ε / Float(c)) * logPreds.mean() 71 | } 72 | -------------------------------------------------------------------------------- /tools/fastai-nbstripout: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import io, sys, argparse, json 4 | 5 | if sys.stdin: input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') 6 | output_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') 7 | 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('-t', '--textconv', action="store_true", help="Print results to output") 10 | parser.add_argument('-d', '--doc-mode', action="store_true", help="fastai docs nb-specific strip out") 11 | parser.add_argument('files', nargs='*', help='Files to strip output from') 12 | args = parser.parse_args() 13 | 14 | # define which fields need to be kept: 15 | cell_metadata_keep_code = [] 16 | cell_metadata_keep_docs = ['hide_input'] 17 | nb_metadata_keep = ['kernelspec', 'jekyll'] 18 | 19 | def clean_cell_outputs(o): 20 | if 'execution_count' in o: o['execution_count'] = None 21 | 22 | ### filter for doc nb cells ### 23 | # 1. reset execution_count (in cell and cell's outputs field) 24 | # 2. keep only cell_metadata_keep_doc fields 25 | 26 | def clean_cell_docs(o): 27 | if 'execution_count' in o: o['execution_count'] = None 28 | if 'outputs' in o: 29 | for l in o['outputs']: clean_cell_outputs(l) 30 | 31 | o['metadata'] = { k:o['metadata'][k] for k in o['metadata'].keys() if k in cell_metadata_keep_docs } 32 | return o 33 | 34 | ### filter for code nb cells ### 35 | # 1. reset execution_count 36 | # 2. delete cell's metadata 37 | # 3. delete cell's outputs 38 | 39 | def clean_cell_code(o): 40 | if 'execution_count' in o: o['execution_count'] = None 41 | if 'outputs' in o: o['outputs'] = [] 42 | o['metadata'] = {} 43 | return o 44 | 45 | # optimize runtime 46 | clean_cell = clean_cell_code if not args.doc_mode else clean_cell_docs 47 | 48 | ### filter for nb top level entries ### 49 | # 1. keep only nb_metadata_keep fields 50 | # 2. the other rules apply based on clean_cell alias 51 | 52 | def clean_nb(s): 53 | s['cells'] = [ clean_cell(o) for o in s['cells'] ] 54 | s['metadata'] = { k:s['metadata'][k] for k in s['metadata'].keys() if k in nb_metadata_keep } 55 | 56 | for filename in args.files: 57 | if not filename.endswith('.ipynb'): continue 58 | with io.open(filename, 'r', encoding='utf-8') as f: s = json.load(f) 59 | clean_nb(s) 60 | x = json.dumps(s, sort_keys=True, indent=1, ensure_ascii=False) 61 | 62 | if args.textconv: 63 | # XXX: if there is more than one file, this is probably wrong 64 | output_stream.write(x) 65 | output_stream.write("\n") 66 | output_stream.flush() 67 | else: 68 | with io.open(filename, 'w', encoding='utf-8') as f: 69 | f.write(x) 70 | f.write("\n") 71 | 72 | # implied textconv mode 73 | if not args.files and input_stream: 74 | try: s = json.load(input_stream) 75 | except: raise Exception(f'failed to load {input_stream}') 76 | clean_nb(s) 77 | x = json.dumps(s, sort_keys=True, indent=1, ensure_ascii=False) 78 | output_stream.write(x) 79 | output_stream.write("\n") 80 | output_stream.flush() 81 | 82 | -------------------------------------------------------------------------------- /Sources/SwiftAI/EarlyStopping.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 05b_early_stopping.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | #if canImport(PythonKit) 15 | import PythonKit 16 | #else 17 | import Python 18 | #endif 19 | 20 | 21 | //cell15 22 | //TODO: when recorder can be accessed as a property, remove it from the return 23 | extension Learner where Opt.Scalar: PythonConvertible { 24 | public func makeDefaultDelegates(metrics: [(Output, Label) -> TF] = []) -> Recorder { 25 | let recorder = makeRecorder() 26 | delegates = [makeTrainEvalDelegate(), makeShowProgress(), recorder] 27 | if !metrics.isEmpty { delegates.append(makeAvgMetric(metrics: metrics)) } 28 | return recorder 29 | } 30 | } 31 | 32 | //cell24 33 | extension Learner where Opt.Scalar: BinaryFloatingPoint { 34 | public class LRFinder: Delegate { 35 | public typealias ScheduleFunc = (Float) -> Float 36 | 37 | // A learning rate schedule from step to float. 38 | private var scheduler: ScheduleFunc 39 | private var numIter: Int 40 | private var minLoss: Float? = nil 41 | 42 | public init(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) { 43 | scheduler = makeAnnealer(start: start, end: end, schedule: expSchedule) 44 | self.numIter = numIter 45 | } 46 | 47 | override public func batchWillStart(learner: Learner) { 48 | learner.opt.learningRate = Opt.Scalar(scheduler(Float(learner.currentIter)/Float(numIter))) 49 | } 50 | 51 | override public func batchDidFinish(learner: Learner) throws { 52 | if minLoss == nil {minLoss = learner.currentLoss.scalar} 53 | else { 54 | if learner.currentLoss.scalarized() < minLoss! { minLoss = learner.currentLoss.scalarized()} 55 | if learner.currentLoss.scalarized() > 4 * minLoss! { 56 | throw LearnerAction.stop(reason: "Loss diverged") 57 | } 58 | if learner.currentIter >= numIter { 59 | throw LearnerAction.stop(reason: "Finished the range.") 60 | } 61 | } 62 | } 63 | 64 | override public func validationWillStart(learner: Learner) throws { 65 | //Skip validation during the LR range test 66 | throw LearnerAction.skipEpoch(reason: "No validation in the LR Finder.") 67 | } 68 | } 69 | 70 | public func makeLRFinder(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) -> LRFinder { 71 | return LRFinder(start: start, end: end, numIter: numIter) 72 | } 73 | } 74 | 75 | //cell29 76 | //TODO: when Recorder is a property of Learner don't return it. 77 | extension Learner where Opt.Scalar: PythonConvertible & BinaryFloatingPoint { 78 | public func lrFind(start: Float = 1e-5, end: Float = 10, numIter: Int = 100) -> Recorder { 79 | let epochCount = data.train.count/numIter + 1 80 | let recorder = makeDefaultDelegates() 81 | delegates.append(makeLRFinder(start: start, end: end, numIter: numIter)) 82 | try! self.fit(epochCount) 83 | return recorder 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SwiftAI 2 | 3 | Pre-alpha version of Swift for TensorFlow's high-level API, modeled after fastai. The *stable* branch works with v0.6 of [Swift for TensorFlow](https://github.com/tensorflow/swift/blob/main/Installation.md) (s4tf), and the *main* branch follows the nightlies of s4tf (unstable and might be broken, see status below). For older releases, use one of the s4tf_\* branch. To learn the details of what's in this repo, check out lessons 13 and 14 of [course.fast.ai](https://course.fast.ai). Full docs will be added here as things become more stable. 4 | 5 | You can download and train [imagenette](https://github.com/fastai/imagenette/) by typing `swift run` at the root of this repo. 6 | 7 | SwiftAI is built from the notebooks in `nbs/`. Once you have the notebooks working the way you want, run the `tools/export_import.ipynb` (we'll be replacing this exporter with a script soon). 8 | 9 | Here's a walk-thru of training a model: 10 | 11 | ```swift 12 | import SwiftAI 13 | import TensorFlow 14 | ``` 15 | 16 | We need both SwiftAI and TensorFlow packages, since SwiftAI is designed to work *with* TensorFlow, not to *replace* TensorFlow. 17 | 18 | ```swift 19 | let path = downloadImagenette() 20 | ``` 21 | 22 | As s4tf and SwiftAI add support for more types of models, we'll be adding lots of datasets; for now just Imagenette and MNIST are provided. 23 | 24 | ```swift 25 | let il = ItemList(fromFolder: path, extensions: ["jpeg", "jpg"]) 26 | let sd = SplitData(il) {grandParentSplitter(fName: $0, valid: "val")} 27 | ``` 28 | 29 | We use the Swift version of the [data block API](https://docs.fast.ai/data_block.html#The-data-block-API) to grab the files we need, and split in to train and validation sets. 30 | 31 | ```swift 32 | var procLabel = CategoryProcessor() 33 | let sld = makeLabeledData(sd, fromFunc: parentLabeler, procLabel: &procLabel) 34 | ``` 35 | 36 | *Processors* are (potentially stateful) functions which preprocess data. In this case, `CategoryProcessor` gets a list of unique labels from the data and creates a mapping to turn labels into `Int`s. 37 | 38 | ```swift 39 | let rawData = sld.toDataBunch(itemToTensor: pathsToTensor, labelToTensor: intsToTensor, bs: 128) 40 | ``` 41 | 42 | A `DataBunch` is a simple object which contains labeled training and validation `Dataset`s. 43 | 44 | ```swift 45 | let data = transformData(rawData) { openAndResize(fname: $0, size: 128) } 46 | ``` 47 | 48 | We can add any lazily-applied transformations we need to convert (for instance) raw file names into data ready for modeling (in this case, images of the same size). 49 | 50 | ```swift 51 | func modelInit() -> XResNet { return xresnet18(cOut: 10) } 52 | let optFunc: (XResNet) -> StatefulOptimizer = adamOpt(lr: 1e-3, mom: 0.9, beta: 0.99, wd: 1e-2, eps: 1e-4) 53 | let learner = Learner(data: data, lossFunc: softmaxCrossEntropy, optFunc: optFunc, modelInit: modelInit) 54 | ``` 55 | 56 | A `Learner` is an object that knows how apply an `Optimizer` (in this case, `adamOpt`, which defaults to [AdamW](https://www.fast.ai/2018/07/02/adam-weight-decay/)) to train a model (`xresnet18`) based on some `DataBunch`, to minimize some differentiable loss function (`softmaxCrossEntropy`). 57 | 58 | ```swift 59 | let recorder = learner.makeDefaultDelegates(metrics: [accuracy]) 60 | learner.addDelegate(learner.makeNormalize(mean: imagenetStats.mean, std: imagenetStats.std)) 61 | learner.addOneCycleDelegates(1e-3, pctStart: 0.5) 62 | ``` 63 | 64 | Delegates are used to customize training in many ways. In this case, we're adding delegates to: 65 | 66 | - Record losses and metrics after each batch, add a progress bar, and move data to the GPU (these are all *default delegates* in SwiftAI) 67 | - Normalize the data 68 | - Use the [1 cycle policy](https://sgugger.github.io/the-1cycle-policy.html) 69 | 70 | ```swift 71 | try! learner.fit(1) 72 | ``` 73 | 74 | The `fit` method will train and validate your model for as many epochs as you request. 75 | 76 | ## Docker 77 | 78 | A Dockerfile has been created to help spin up a CPU based learning and development environment. 79 | 80 | ```sh 81 | # Only Build image 82 | make 83 | 84 | # Build image and run Jupyter 85 | make jupyter 86 | 87 | # Build image and run shell 88 | make shell 89 | 90 | # Sync notebooks to sources (Sources must not have modifications) 91 | make sync-nbs-to-srcs 92 | ``` 93 | 94 | ## Status 95 | 96 | Master is currently working with the latest nightlies toolchain. 97 | -------------------------------------------------------------------------------- /tools/trust-origin-git-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # This script facilitates trusting/distrusting of the repo-wide .gitconfig 4 | # 5 | # trust: 6 | # tools/trust-origin-git-config 7 | # or 8 | # tools/trust-origin-git-config -e 9 | # which is the same as: 10 | # git config --local include.path ../.gitconfig 11 | # 12 | # distrust: 13 | # tools/trust-origin-git-config -d 14 | # which is the same as: 15 | # git config --local --unset include.path 16 | # 17 | # note: windows users, not using bash emulation, will need to invoke this tool as: 18 | # python tools\trust-origin-git-config 19 | 20 | import sys 21 | if sys.hexversion < 0x03060000: sys.exit("!!! Please re-run this script with python-3.6 or higher") 22 | 23 | import os, argparse, subprocess 24 | 25 | parser = argparse.ArgumentParser() 26 | parser.add_argument('-e', '--enable', action="store_true", help="Trust repo-wide .gitconfig (default action)") 27 | parser.add_argument('-d', '--disable', action="store_true", help="Distrust repo-wide .gitconfig") 28 | parser.add_argument('-t', '--test', action="store_true", help="Validate repo-wide .gitconfig config") 29 | args = parser.parse_args() 30 | 31 | 32 | # test exec bit 33 | def validate_script(): 34 | filepath = os.path.join('tools', 'fastai-nbstripout') 35 | 36 | # check that we can execute the script 37 | cmd = f"{filepath} -h" 38 | #print(f"Executing: {cmd}") 39 | result = subprocess.run(cmd.split(), shell=False, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 40 | # we don't care for the output, just to see that it works 41 | if result.returncode != 0: 42 | print(f"Can't execute {filepath}") 43 | if result.stderr: print(f"Error: {result.stderr.decode('utf-8')}") 44 | 45 | def write_config(): 46 | is_windows = hasattr(sys, 'getwindowsversion') 47 | cmd = "tools/fastai-nbstripout" if not is_windows else r"python tools\\\\fastai-nbstripout" 48 | with open(".gitconfig", 'w') as f: 49 | f.write(f"""# You need to enable this configuration once after checking out the repo 50 | # for the first time (assuming you checked it out into the swiftai/ dir): 51 | # 52 | # cd swiftai 53 | # git config --local include.path ../.gitconfig 54 | # 55 | # If you need to disable this instrumentation do: 56 | # 57 | # git config --local --unset include.path 58 | # 59 | # You can always check .git/config to see whether a ../.gitconfig 60 | # [include] entry is there or not. 61 | # 62 | # If tools/fastai-nbstripout is modified to produce a different output, 63 | # manually rerun all the notebooks under git: 64 | # 65 | # {cmd} -d nbs/*/*ipynb 66 | # 67 | # # disable the strip out filter to get git to see changes 68 | # git config --local --unset include.path 69 | # git commit -a 70 | # git push 71 | # # restore the filter 72 | # git config --local include.path ../.gitconfig 73 | # 74 | 75 | # code notebooks strip out 76 | [filter "fastai-nbstripout-code"] 77 | clean = {cmd} 78 | smudge = cat 79 | required = true 80 | [diff "ipynb-code"] 81 | textconv = {cmd} -t 82 | 83 | # docs notebooks strip out 84 | [filter "fastai-nbstripout-docs"] 85 | clean = {cmd} -d 86 | smudge = cat 87 | required = true 88 | [diff "ipynb-docs"] 89 | textconv = {cmd} -dt 90 | """) 91 | 92 | def trust_enable(): 93 | #validate_script() 94 | 95 | write_config() 96 | cmd = "git config --local include.path ../.gitconfig" 97 | print(f"Executing: {cmd}") 98 | result = subprocess.run(cmd.split(), shell=False, check=False, stderr=subprocess.PIPE) 99 | if result.returncode == 0: 100 | print("Success: repo's .gitconfig is now trusted") 101 | else: 102 | print("Failed to trust repo's .gitconfig") 103 | if result.stderr: print(f"Error: {result.stderr.decode('utf-8')}") 104 | 105 | 106 | def trust_disable(): 107 | 108 | cmd = "git config --local --unset include.path" 109 | print(f"Executing: {cmd}") 110 | result = subprocess.run(cmd.split(), shell=False, check=False, stderr=subprocess.PIPE) 111 | if result.returncode == 0: 112 | print("Success: repo's .gitconfig is now distrusted") 113 | else: 114 | print("Failed to distrust repo's .gitconfig") 115 | if result.stderr: print(f"Error: {result.stderr.decode('utf-8')}") 116 | 117 | def trust_test(): 118 | #validate_script() 119 | 120 | cmd = "git config --list --show-origin" 121 | print(f"Executing: {cmd}") 122 | result = subprocess.run(cmd.split(), shell=False, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 123 | if result.returncode == 0: 124 | out = result.stdout.decode('utf-8') 125 | if ".git/../.gitconfig" in out and "filter.fastai-nbstripout-code" in out: 126 | print("Check: repo's .gitconfig is trusted") 127 | else: 128 | print("Check: repo's .gitconfig is not trusted, re-run with -e option to trust it") 129 | else: 130 | print(f"Failed to run {cmd}") 131 | if result.stderr: print(f"Error: {result.stderr.decode('utf-8')}") 132 | 133 | 134 | 135 | if args.test: trust_test() 136 | elif args.disable: trust_disable() 137 | else: trust_enable() 138 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Imagenette.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 11_imagenette.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | import TensorFlow 14 | 15 | //cell25 16 | public struct ConvLayer: Layer { 17 | public var bn: FABatchNorm 18 | public var conv: FANoBiasConv2D 19 | 20 | public init(_ cIn: Int, _ cOut: Int, ks: Int = 3, stride: Int = 1, zeroBn: Bool = false, act: Bool = true){ 21 | bn = FABatchNorm(featureCount: cOut) 22 | // "activation: act ? relu : identity" fails on 0.3.1, so we use if/else 23 | if act {conv = FANoBiasConv2D(cIn, cOut, ks: ks, stride: stride, activation: relu)} 24 | else {conv = FANoBiasConv2D(cIn, cOut, ks: ks, stride: stride, activation: identity)} 25 | if zeroBn { bn.scale = Tensor(zeros: [cOut]) } 26 | } 27 | 28 | @differentiable 29 | public func callAsFunction(_ input: TF) -> TF { 30 | // TODO: Work around https://bugs.swift.org/browse/TF-606 31 | return bn.forward(conv.forward(input)) 32 | } 33 | } 34 | 35 | //cell27 36 | public struct MaybeAvgPool2D: ParameterlessLayer { 37 | @noDerivative let poolSize: (Int, Int, Int, Int) 38 | @noDerivative let strides: (Int, Int, Int, Int) 39 | @noDerivative let padding: Padding 40 | @noDerivative public var isOn: Bool 41 | 42 | @differentiable public func callAsFunction(_ input: TF) -> TF { 43 | return isOn ? avgPool2D(input, filterSize: poolSize, strides: strides, padding: padding) : input 44 | } 45 | 46 | public init(_ sz: Int, padding: Padding = .valid) { 47 | isOn = (sz>1) 48 | poolSize = (1, sz, sz, 1) 49 | strides = (1, sz, sz, 1) 50 | self.padding = padding 51 | } 52 | } 53 | 54 | //cell28 55 | public struct MaybeConv: Layer { 56 | var conv: ConvLayer 57 | @noDerivative public var isOn: Bool 58 | 59 | @differentiable public func callAsFunction(_ input: TF) -> TF { 60 | return isOn ? conv(input) : input 61 | } 62 | 63 | public init(_ cIn: Int, _ cOut: Int) { 64 | isOn = (cIn > 1) || (cOut > 1) 65 | conv = ConvLayer(cIn, cOut, ks: 1, act: false) 66 | } 67 | } 68 | 69 | //cell30 70 | public struct ResBlock: Layer { 71 | public var convs: [ConvLayer] 72 | public var idConv: MaybeConv 73 | public var pool: MaybeAvgPool2D 74 | 75 | public init(_ expansion: Int, _ ni: Int, _ nh: Int, stride: Int = 1){ 76 | let (nf, nin) = (nh*expansion,ni*expansion) 77 | convs = (expansion==1) ? [ 78 | ConvLayer(nin, nh, ks: 3, stride: stride), 79 | ConvLayer(nh, nf, ks: 3, zeroBn: true, act: false) 80 | ] : [ 81 | ConvLayer(nin, nh, ks: 1), 82 | ConvLayer(nh, nh, ks: 3, stride: stride), 83 | ConvLayer(nh, nf, ks: 1, zeroBn: true, act: false) 84 | ] 85 | idConv = nin==nf ? MaybeConv(1,1) : MaybeConv(nin, nf) 86 | pool = MaybeAvgPool2D(stride) 87 | } 88 | 89 | @differentiable 90 | public func callAsFunction(_ inp: TF) -> TF { 91 | return relu(convs(inp) + idConv(pool(inp))) 92 | } 93 | 94 | } 95 | 96 | //cell32 97 | func makeLayer(_ expansion: Int, _ ni: Int, _ nf: Int, _ nBlocks: Int, stride: Int) -> [ResBlock] { 98 | return Array(0..(poolSize: (3,3), strides: (2,2), padding: .same) 105 | public var blocks: [ResBlock] 106 | public var pool = GlobalAvgPool2D() 107 | public var linear: Dense 108 | 109 | public init(_ expansion: Int, _ layers: [Int], cIn: Int = 3, cOut: Int = 1000){ 110 | var nfs = [cIn, (cIn+1)*8, 64, 64] 111 | stem = (0..<3).map{ ConvLayer(nfs[$0], nfs[$0+1], stride: $0==0 ? 2 : 1)} 112 | nfs = [64/expansion,64,128,256,512] 113 | blocks = layers.enumerated().map { (i,l) in 114 | return makeLayer(expansion, nfs[i], nfs[i+1], l, stride: i==0 ? 1 : 2) 115 | }.reduce([], +) 116 | linear = Dense(inputSize: nfs.last!*expansion, outputSize: cOut) 117 | } 118 | 119 | @differentiable 120 | public func callAsFunction(_ inp: TF) -> TF { 121 | return inp.compose(stem, maxPool, blocks, pool, linear) 122 | } 123 | } 124 | 125 | //cell34 126 | public func xresnet18 (cIn: Int = 3, cOut: Int = 1000) -> XResNet { return XResNet(1, [2, 2, 2, 2], cIn: cIn, cOut: cOut) } 127 | public func xresnet34 (cIn: Int = 3, cOut: Int = 1000) -> XResNet { return XResNet(1, [3, 4, 6, 3], cIn: cIn, cOut: cOut) } 128 | public func xresnet50 (cIn: Int = 3, cOut: Int = 1000) -> XResNet { return XResNet(4, [3, 4, 6, 3], cIn: cIn, cOut: cOut) } 129 | public func xresnet101(cIn: Int = 3, cOut: Int = 1000) -> XResNet { return XResNet(4, [3, 4, 23, 3], cIn: cIn, cOut: cOut) } 130 | public func xresnet152(cIn: Int = 3, cOut: Int = 1000) -> XResNet { return XResNet(4, [3, 8, 36, 3], cIn: cIn, cOut: cOut) } 131 | -------------------------------------------------------------------------------- /Sources/SwiftAI/LoadData.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 00_load_data.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | precedencegroup ExponentiationPrecedence { 13 | associativity: right 14 | higherThan: MultiplicationPrecedence 15 | } 16 | infix operator ** : ExponentiationPrecedence 17 | 18 | precedencegroup CompositionPrecedence { associativity: left } 19 | infix operator >| : CompositionPrecedence 20 | 21 | //cell4 22 | import Foundation 23 | import Just 24 | import Path 25 | 26 | //cell6 27 | public extension String { 28 | @discardableResult 29 | func shell(_ args: String...) -> String 30 | { 31 | let (task,pipe) = (Process(),Pipe()) 32 | task.executableURL = URL(fileURLWithPath: self) 33 | (task.arguments,task.standardOutput) = (args,pipe) 34 | do { try task.run() } 35 | catch { print("Unexpected error: \(error).") } 36 | 37 | let data = pipe.fileHandleForReading.readDataToEndOfFile() 38 | return String(data: data, encoding: String.Encoding.utf8) ?? "" 39 | } 40 | } 41 | 42 | //cell9 43 | public func downloadFile(_ url: String, dest: String? = nil, force: Bool = false) { 44 | let dest_name = dest ?? (Path.cwd/url.split(separator: "/").last!).string 45 | let url_dest = URL(fileURLWithPath: (dest ?? (Path.cwd/url.split(separator: "/").last!).string)) 46 | if !force && Path(dest_name)!.exists { return } 47 | 48 | print("Downloading \(url)...") 49 | 50 | if let cts = Just.get(url).content { 51 | do {try cts.write(to: URL(fileURLWithPath:dest_name))} 52 | catch {print("Can't write to \(url_dest).\n\(error)")} 53 | } else { 54 | print("Can't reach \(url)") 55 | } 56 | } 57 | 58 | //cell12 59 | import TensorFlow 60 | 61 | //cell16 62 | protocol ConvertibleFromByte: TensorFlowScalar { 63 | init(_ d:UInt8) 64 | } 65 | 66 | //cell18 67 | extension Float : ConvertibleFromByte {} 68 | extension Int32 : ConvertibleFromByte {} 69 | 70 | //cell20 71 | extension Data { 72 | func asTensor() -> Tensor { 73 | return Tensor(map(T.init)) 74 | } 75 | } 76 | 77 | //cell22 78 | func loadMNIST 79 | (training: Bool, labels: Bool, path: Path, flat: Bool) -> Tensor { 80 | let split = training ? "train" : "t10k" 81 | let kind = labels ? "labels" : "images" 82 | let batch = training ? 60000 : 10000 83 | let shape: TensorShape = labels ? [batch] : (flat ? [batch, 784] : [batch, 28, 28]) 84 | let dropK = labels ? 8 : 16 85 | let baseUrl = "https://storage.googleapis.com/cvdf-datasets/mnist/" 86 | let fname = split + "-" + kind + "-idx\(labels ? 1 : 3)-ubyte" 87 | let file = path/fname 88 | if !file.exists { 89 | downloadFile("\(baseUrl)\(fname).gz", dest:(path/"\(fname).gz").string) 90 | "/bin/gunzip".shell("-fq", (path/"\(fname).gz").string) 91 | } 92 | let data = try! Data(contentsOf: URL(fileURLWithPath: file.string)).dropFirst(dropK) 93 | if labels { return data.asTensor() } 94 | else { return data.asTensor().reshaped(to: shape)} 95 | } 96 | 97 | public func loadMNIST(path:Path, flat:Bool = false) 98 | -> (Tensor, Tensor, Tensor, Tensor) { 99 | try! path.mkdir(.p) 100 | return ( 101 | loadMNIST(training: true, labels: false, path: path, flat: flat) / 255.0, 102 | loadMNIST(training: true, labels: true, path: path, flat: flat), 103 | loadMNIST(training: false, labels: false, path: path, flat: flat) / 255.0, 104 | loadMNIST(training: false, labels: true, path: path, flat: flat) 105 | ) 106 | } 107 | 108 | //cell24 109 | public let mnistPath = Path.home/".fastai"/"data"/"mnist_tst" 110 | 111 | //cell31 112 | import Dispatch 113 | 114 | // ⏰Time how long it takes to run the specified function, optionally taking 115 | // the average across a number of repetitions. 116 | public func time(repeating: Int = 1, _ f: () -> ()) { 117 | guard repeating > 0 else { return } 118 | 119 | // Warmup 120 | if repeating > 1 { f() } 121 | 122 | var times = [Double]() 123 | for _ in 1...repeating { 124 | let start = DispatchTime.now() 125 | f() 126 | let end = DispatchTime.now() 127 | let nanoseconds = Double(end.uptimeNanoseconds - start.uptimeNanoseconds) 128 | let milliseconds = nanoseconds / 1e6 129 | times.append(milliseconds) 130 | } 131 | print("average: \(times.reduce(0.0, +)/Double(times.count)) ms, " + 132 | "min: \(times.reduce(times[0], min)) ms, " + 133 | "max: \(times.reduce(times[0], max)) ms") 134 | } 135 | 136 | //cell35 137 | public extension String { 138 | func findFirst(pat: String) -> Range? { 139 | return range(of: pat, options: .regularExpression) 140 | } 141 | func hasMatch(pat: String) -> Bool { 142 | return findFirst(pat:pat) != nil 143 | } 144 | } 145 | 146 | //cell38 147 | public func notebookToScript(fname: Path){ 148 | let newname = fname.basename(dropExtension: true)+".swift" 149 | let url = fname.parent/"FastaiNotebooks/Sources/FastaiNotebooks"/newname 150 | do { 151 | let data = try Data(contentsOf: fname.url) 152 | let jsonData = try JSONSerialization.jsonObject(with: data, options: .allowFragments) as! [String: Any] 153 | let cells = jsonData["cells"] as! [[String:Any]] 154 | var module = """ 155 | /* 156 | THIS FILE WAS AUTOGENERATED! DO NOT EDIT! 157 | file to edit: \(fname.lastPathComponent) 158 | 159 | */ 160 | 161 | """ 162 | for cell in cells { 163 | if let source = cell["source"] as? [String], !source.isEmpty, 164 | source[0].hasMatch(pat: #"^\s*//\s*export\s*$"#) { 165 | module.append("\n" + source[1...].joined() + "\n") 166 | } 167 | } 168 | try module.write(to: url, encoding: .utf8) 169 | } catch { 170 | print("Can't read the content of \(fname)") 171 | } 172 | } 173 | 174 | //cell40 175 | public func exportNotebooks(_ path: Path) { 176 | for entry in try! path.ls() 177 | where entry.kind == Entry.Kind.file && 178 | entry.path.basename().hasMatch(pat: #"^\d*_.*ipynb$"#) { 179 | print("Converting \(entry)") 180 | notebookToScript(fname: entry.path) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Anneal.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 05_anneal.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | 15 | //cell16 16 | #if canImport(PythonKit) 17 | import PythonKit 18 | #else 19 | import Python 20 | #endif 21 | public let np = Python.import("numpy") 22 | public let plt = Python.import("matplotlib.pyplot") 23 | 24 | //cell17 25 | public func plot(_ arr1: [S1], _ arr2: [S2], logScale:Bool = false, xLabel: String="", yLabel: String = "") 26 | where S1:PythonConvertible, S2:PythonConvertible{ 27 | plt.figure(figsize: [6,4]) 28 | let (npArr1, npArr2) = (np.array(arr1), np.array(arr2)) 29 | if logScale {plt.xscale("log")} 30 | if !xLabel.isEmpty {plt.xlabel(xLabel)} 31 | if !yLabel.isEmpty {plt.ylabel(yLabel)} 32 | let fig = plt.plot(npArr1, npArr2) 33 | plt.show(fig) 34 | } 35 | 36 | //cell18 37 | extension Learner where Opt.Scalar: PythonConvertible{ 38 | public class Recorder: Delegate { 39 | public var losses: [Loss] = [] 40 | public var lrs: [Opt.Scalar] = [] 41 | 42 | public override func batchDidFinish(learner: Learner) { 43 | if learner.inTrain { 44 | losses.append(learner.currentLoss) 45 | lrs.append(learner.opt.learningRate) 46 | } 47 | } 48 | 49 | public func plotLosses(){ 50 | plot(Array(0.. Recorder { 64 | return Recorder() 65 | } 66 | } 67 | 68 | //cell27 69 | import Foundation 70 | 71 | //cell28 72 | func formatTime(_ t: Float) -> String { 73 | let t = Int(t) 74 | let (h,m,s) = (t/3600, (t/60)%60, t%60) 75 | return h != 0 ? String(format: "%02d:%02d:%02d", h, m, s) : String(format: "%02d:%02d", m, s) 76 | } 77 | 78 | //cell30 79 | public struct ProgressBar{ 80 | let total: Int 81 | let length: Int = 50 82 | let showEvery: Float = 0.2 83 | let fillChar: Character = "X" 84 | public var comment: String = "" 85 | private var waitFor: Int = 0 86 | private var startTime: UInt64 = 0 87 | private var lastPrint: UInt64 = 0 88 | private var lastShow: UInt64 = 0 89 | private var estimatedTotal: Float = 0.0 90 | private var bar: String = "" 91 | 92 | public init(_ c: Int) { total = c } 93 | 94 | public mutating func update(_ val: Int){ 95 | lastShow = DispatchTime.now().uptimeNanoseconds 96 | if val == 0 { startTime = lastShow } 97 | else { 98 | let averageTime = Float(lastShow - startTime) / (1e9 * Float(val)) 99 | estimatedTotal = Float(total) * averageTime 100 | } 101 | if val == 0 || lastShow - lastPrint >= Int(1e9 * showEvery) { update_bar(val) } 102 | } 103 | 104 | public mutating func update_bar(_ val: Int){ 105 | lastPrint = lastShow 106 | let prevLength = bar.count 107 | bar = String(repeating: fillChar, count: (val * length) / total) 108 | bar += String(repeating: "-", count: length - (val * length) / total) 109 | let pct = String(format: "%.2f", 100.0 * Float(val)/Float(total)) 110 | let elapsedTime = Float(lastShow - startTime) / 1e9 111 | let remaingTime = estimatedTotal - elapsedTime 112 | bar += " \(pct)% [\(val)/\(total) \(formatTime(elapsedTime))<\(formatTime(remaingTime))" 113 | bar += comment.isEmpty ? "]" : " \(comment)]" 114 | if bar.count < prevLength { bar += String(repeating: " ", count: prevLength-bar.count) } 115 | print(bar, terminator:"\r") 116 | fflush(stdout) 117 | } 118 | 119 | public func remove(){ 120 | print(String(repeating: " ", count: bar.count), terminator:"\r") 121 | fflush(stdout) 122 | } 123 | } 124 | 125 | //cell32 126 | extension Learner { 127 | public class ShowProgress: Delegate { 128 | var pbar: ProgressBar? = nil 129 | var iter: Int = 0 130 | 131 | public override func epochWillStart(learner: Learner) { 132 | pbar = ProgressBar(learner.data.train.count) 133 | } 134 | 135 | public override func validationWillStart(learner: Learner) { 136 | if pbar != nil { pbar!.remove() } 137 | pbar = ProgressBar(learner.data.valid.count) 138 | } 139 | 140 | public override func epochDidFinish(learner: Learner) { 141 | if pbar != nil { pbar!.remove() } 142 | } 143 | 144 | public override func batchWillStart(learner: Learner) { 145 | if learner.currentIter == 0 {pbar!.update(0)} 146 | } 147 | 148 | public override func batchDidFinish(learner: Learner) { 149 | pbar!.update(learner.currentIter) 150 | } 151 | 152 | public override func trainingDidFinish(learner: Learner) { 153 | if pbar != nil { pbar!.remove() } 154 | } 155 | } 156 | 157 | public func makeShowProgress() -> ShowProgress { return ShowProgress() } 158 | } 159 | 160 | //cell37 161 | /// A non-generalized learning rate scheduler 162 | extension Learner where Opt.Scalar: BinaryFloatingPoint { 163 | public class LRScheduler: Delegate { 164 | public override var order: Int { return 1 } 165 | public typealias ScheduleFunc = (Float) -> Float 166 | 167 | // A learning rate schedule from step to float. 168 | public var scheduler: ScheduleFunc 169 | 170 | public init(scheduler: @escaping (Float) -> Float) { 171 | self.scheduler = scheduler 172 | } 173 | 174 | override public func batchWillStart(learner: Learner) { 175 | learner.opt.learningRate = Opt.Scalar(scheduler(learner.pctEpochs/Float(learner.epochCount))) 176 | } 177 | } 178 | 179 | public func makeLRScheduler(scheduler: @escaping (Float) -> Float) -> LRScheduler { 180 | return LRScheduler(scheduler: scheduler) 181 | } 182 | } 183 | 184 | //cell38 185 | public func linearSchedule(start: Float, end: Float, pct: Float) -> Float { 186 | return start + pct * (end - start) 187 | } 188 | 189 | public func makeAnnealer(start: Float, end: Float, schedule: @escaping (Float, Float, Float) -> Float) -> (Float) -> Float { 190 | return { pct in return schedule(start, end, pct) } 191 | } 192 | 193 | //cell45 194 | public func constantSchedule(start: Float, end: Float, pct: Float) -> Float { 195 | return start 196 | } 197 | 198 | public func cosineSchedule(start: Float, end: Float, pct: Float) -> Float { 199 | return start + (1 + cos(Float.pi*(1-pct))) * (end-start) / 2 200 | } 201 | 202 | public func expSchedule(start: Float, end: Float, pct: Float) -> Float { 203 | return start * pow(end / start, pct) 204 | } 205 | 206 | //cell46 207 | public func combineSchedules(pcts: [Float], schedules: [(Float) -> Float]) -> ((Float) -> Float){ 208 | var cumPcts: [Float] = [0] 209 | for pct in pcts {cumPcts.append(cumPcts.last! + pct)} 210 | func inner(pct: Float) -> Float{ 211 | if (pct == 0.0) { return schedules[0](0.0) } 212 | if (pct > 1.0) { return schedules.last!(1.0) } 213 | let i = cumPcts.firstIndex(where: {$0 >= pct})! - 1 214 | let actualPos = (pct-cumPcts[i]) / (cumPcts[i+1]-cumPcts[i]) 215 | return schedules[i](actualPos) 216 | } 217 | return inner 218 | } 219 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Batchnorm.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 07_batchnorm.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | import TensorFlow 14 | #if canImport(PythonKit) 15 | import PythonKit 16 | #else 17 | import Python 18 | #endif 19 | 20 | 21 | //cell13 22 | public class Reference { 23 | public var value: T 24 | public init(_ value: T) { self.value = value } 25 | } 26 | 27 | //cell15 28 | public protocol LearningPhaseDependent: FALayer { 29 | associatedtype Input 30 | associatedtype Output 31 | 32 | @differentiable func forwardTraining(_ input: Input) -> Output 33 | @differentiable func forwardInference(_ input: Input) -> Output 34 | } 35 | 36 | extension LearningPhaseDependent { 37 | @differentiable 38 | public func forward(_ input: Input) -> Output { 39 | switch Context.local.learningPhase { 40 | case .training: return forwardTraining(input) 41 | case .inference: return forwardInference(input) 42 | } 43 | } 44 | } 45 | 46 | //cell17 47 | public protocol Norm: Layer where Input == Tensor, Output == Tensor{ 48 | associatedtype Scalar 49 | init(featureCount: Int, epsilon: Scalar) 50 | } 51 | 52 | public struct FABatchNorm: LearningPhaseDependent, Norm { 53 | // TF-603 workaround. 54 | public typealias Input = Tensor 55 | public typealias Output = Tensor 56 | @noDerivative public var delegates: [(Self.Output) -> ()] = [] 57 | 58 | // Configuration hyperparameters 59 | @noDerivative var momentum, epsilon: Scalar 60 | // Running statistics 61 | @noDerivative let runningMean, runningVariance: Reference> 62 | // Trainable parameters 63 | public var scale, offset: Tensor 64 | 65 | public init(featureCount: Int, momentum: Scalar, epsilon: Scalar = 1e-5) { 66 | self.momentum = momentum 67 | self.epsilon = epsilon 68 | self.scale = Tensor(ones: [featureCount]) 69 | self.offset = Tensor(zeros: [featureCount]) 70 | self.runningMean = Reference(Tensor(0)) 71 | self.runningVariance = Reference(Tensor(1)) 72 | } 73 | 74 | public init(featureCount: Int, epsilon: Scalar = 1e-5) { 75 | self.init(featureCount: featureCount, momentum: 0.9, epsilon: epsilon) 76 | } 77 | 78 | @differentiable 79 | public func forwardTraining(_ input: Tensor) -> Tensor { 80 | let mean = input.mean(alongAxes: [0, 1, 2]) 81 | let variance = input.variance(alongAxes: [0, 1, 2]) 82 | runningMean.value += (mean - runningMean.value) * (1 - momentum) 83 | runningVariance.value += (variance - runningVariance.value) * (1 - momentum) 84 | let normalizer = rsqrt(variance + epsilon) * scale 85 | return (input - mean) * normalizer + offset 86 | } 87 | 88 | @differentiable 89 | public func forwardInference(_ input: Tensor) -> Tensor { 90 | let mean = runningMean.value 91 | let variance = runningVariance.value 92 | let normalizer = rsqrt(variance + epsilon) * scale 93 | return (input - mean) * normalizer + offset 94 | } 95 | } 96 | 97 | //cell19 98 | struct BatchNormResult : Differentiable{ 99 | var y, batchMean, batchVariance, reserveSpace1, reserveSpace2: Tensor 100 | } 101 | 102 | public struct TFBatchNorm: LearningPhaseDependent, Norm { 103 | // TF-920 workaround. 104 | public typealias Input = Tensor 105 | public typealias Output = Tensor 106 | @noDerivative public var delegates: [(Self.Output) -> ()] = [] 107 | 108 | // Configuration hyperparameters 109 | @noDerivative var momentum, epsilon: Scalar 110 | // Running statistics 111 | @noDerivative let runningMean, runningVariance: Reference> 112 | // Trainable parameters 113 | public var scale, offset: Tensor 114 | 115 | public init(featureCount: Int, momentum: Scalar, epsilon: Scalar = 1e-5) { 116 | self.momentum = momentum 117 | self.epsilon = epsilon 118 | self.scale = Tensor(ones: [featureCount]) 119 | self.offset = Tensor(zeros: [featureCount]) 120 | self.runningMean = Reference(Tensor(0)) 121 | self.runningVariance = Reference(Tensor(1)) 122 | } 123 | 124 | public init(featureCount: Int, epsilon: Scalar = 1e-5) { 125 | self.init(featureCount: featureCount, momentum: 0.9, epsilon: epsilon) 126 | } 127 | 128 | @differentiable 129 | public func forwardTraining(_ input: Tensor) -> Tensor { 130 | let res = TFBatchNorm.fusedBatchNorm( 131 | input, scale: scale, offset: offset, epsilon: epsilon) 132 | let (output, mean, variance) = (res.y, res.batchMean, res.batchVariance) 133 | runningMean.value += (mean - runningMean.value) * (1 - momentum) 134 | runningVariance.value += (variance - runningVariance.value) * (1 - momentum) 135 | return output 136 | } 137 | 138 | @differentiable 139 | public func forwardInference(_ input: Tensor) -> Tensor { 140 | let mean = runningMean.value 141 | let variance = runningVariance.value 142 | let normalizer = rsqrt(variance + epsilon) * scale 143 | return (input - mean) * normalizer + offset 144 | } 145 | 146 | @differentiable(wrt: (x, scale, offset)) 147 | static func fusedBatchNorm( 148 | _ x : Tensor, scale: Tensor, offset: Tensor, epsilon: Scalar 149 | ) -> BatchNormResult { 150 | let ret = _Raw.fusedBatchNormV2( 151 | x, scale: scale, offset: offset, 152 | mean: Tensor([] as [Scalar]), variance: Tensor([] as [Scalar]), 153 | epsilon: Double(epsilon)) 154 | return BatchNormResult( 155 | y: ret.y, batchMean: ret.batchMean, batchVariance: ret.batchVariance, 156 | reserveSpace1: ret.reserveSpace1, reserveSpace2: ret.reserveSpace2 157 | ) 158 | } 159 | 160 | @derivative(of: fusedBatchNorm, wrt: (x, scale, offset)) 161 | static func _vjpFusedBatchNorm( 162 | _ x : Tensor, scale: Tensor, offset: Tensor, epsilon: Scalar 163 | ) -> ( 164 | value: BatchNormResult, 165 | pullback: (BatchNormResult.TangentVector) -> ( 166 | Tensor.TangentVector, 167 | Tensor.TangentVector, 168 | Tensor.TangentVector 169 | ) 170 | ) { 171 | let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon) 172 | return ( 173 | bnresult, 174 | {v in 175 | let res = _Raw.fusedBatchNormGradV2( 176 | yBackprop: v.y, x, scale: Tensor(scale), 177 | reserveSpace1: bnresult.reserveSpace1, 178 | reserveSpace2: bnresult.reserveSpace2, 179 | epsilon: Double(epsilon)) 180 | return (res.xBackprop, res.scaleBackprop, res.offsetBackprop) 181 | }) 182 | } 183 | } 184 | 185 | //cell20 186 | public struct ConvBN: FALayer { 187 | // TF-603 workaround. 188 | public typealias Input = Tensor 189 | public typealias Output = Tensor 190 | @noDerivative public var delegates: [(Self.Output) -> ()] = [] 191 | public var conv: FANoBiasConv2D 192 | public var norm: FABatchNorm 193 | 194 | public init(_ cIn: Int, _ cOut: Int, ks: Int = 3, stride: Int = 1){ 195 | // TODO (when control flow AD works): use Conv2D without bias 196 | self.conv = FANoBiasConv2D(cIn, cOut, ks: ks, stride: stride, activation: relu) 197 | self.norm = FABatchNorm(featureCount: cOut, epsilon: 1e-5) 198 | } 199 | 200 | @differentiable 201 | public func forward(_ input: Tensor) -> Tensor { 202 | return norm.forward(conv.forward(input)) 203 | } 204 | } 205 | 206 | //cell22 207 | public struct CnnModelBN: Layer { 208 | // TF-944 workaround 209 | public typealias Input = TF 210 | public typealias Output = TF 211 | 212 | public var convs: [ConvBN] 213 | public var pool = FAGlobalAvgPool2D() 214 | public var linear: FADense 215 | @noDerivative public var delegates: [(Self.Output) -> ()] = [] 216 | 217 | public init(channelIn: Int, nOut: Int, filters: [Int]){ 218 | let allFilters = [channelIn] + filters 219 | convs = Array(0..(filters.last!, nOut) 223 | } 224 | 225 | @differentiable 226 | public func callAsFunction(_ input: TF) -> TF { 227 | return linear(pool(convs(input))) 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Sources/SwiftAI/DataBlock.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 08_data_block.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | #if canImport(PythonKit) 15 | import PythonKit 16 | #else 17 | import Python 18 | #endif 19 | 20 | 21 | //cell8 22 | public let dataPath = Path.home/".fastai"/"data" 23 | 24 | //cell9 25 | public func downloadImagenette(path: Path = dataPath, sz:String="-160") -> Path { 26 | let url = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette\(sz).tgz" 27 | let fname = "imagenette\(sz)" 28 | let file = path/fname 29 | try! path.mkdir(.p) 30 | if !file.exists { 31 | downloadFile(url, dest:(path/"\(fname).tgz").string) 32 | _ = "/usr/bin/env".shell("tar", "-xzf", (path/"\(fname).tgz").string, "-C", path.string) 33 | } 34 | return file 35 | } 36 | 37 | //cell21 38 | public func showImg(_ img: Tensor, _ w: Int = 7, _ h: Int = 5) { 39 | showImg(img.makeNumpyArray(), w, h) 40 | } 41 | 42 | public func showImg(_ img: PythonObject, _ w: Int = 7, _ h: Int = 5) { 43 | plt.figure(figsize: [w, h]) 44 | plt.imshow(img) 45 | plt.axis("off") 46 | plt.show() 47 | } 48 | 49 | //cell25 50 | public func fetchFiles(path: Path, recurse: Bool = false, extensions: [String]? = nil) -> [Path] { 51 | var res: [Path] = [] 52 | for p in try! path.ls(){ 53 | if p.kind == .directory && recurse { 54 | res += fetchFiles(path: p.path, recurse: recurse, extensions: extensions) 55 | } else if extensions == nil || extensions!.contains(p.path.extension.lowercased()) { 56 | res.append(p.path) 57 | } 58 | } 59 | return res 60 | } 61 | 62 | //cell32 63 | public struct ItemList{ 64 | public var items: [Item] 65 | public let path: Path 66 | 67 | public init(items: [Item], path: Path){ 68 | (self.items,self.path) = (items,path) 69 | } 70 | } 71 | 72 | //cell33 73 | public extension ItemList where Item == Path { 74 | init(fromFolder path: Path, extensions: [String], recurse: Bool = true) { 75 | self.init(items: fetchFiles(path: path, recurse: recurse, extensions: extensions), 76 | path: path) 77 | } 78 | } 79 | 80 | //cell36 81 | public struct SplitData{ 82 | public let train: ItemList 83 | public let valid: ItemList 84 | public var path: Path { return train.path } 85 | 86 | public init(train: ItemList, valid: ItemList){ 87 | (self.train, self.valid) = (train, valid) 88 | } 89 | 90 | public init(_ il: ItemList, fromFunc: (Item) -> Bool){ 91 | self.init(train: ItemList(items: il.items.filter { !fromFunc($0) }, path: il.path), 92 | valid: ItemList(items: il.items.filter { fromFunc($0) }, path: il.path)) 93 | } 94 | } 95 | 96 | //cell37 97 | public func grandParentSplitter(fName: Path, valid: String = "valid") -> Bool{ 98 | return fName.parent.parent.basename() == valid 99 | } 100 | 101 | //cell40 102 | public protocol Processor { 103 | associatedtype Input 104 | associatedtype Output 105 | 106 | mutating func initState(items: [Input]) 107 | func process1(item: Input) -> Output 108 | func deprocess1(item: Output) -> Input 109 | } 110 | 111 | //cell41 112 | public extension Processor { 113 | func process(items: [Input]) -> [Output] { 114 | return items.map { process1(item: $0) } 115 | } 116 | 117 | func deprocess(items: [Output]) -> [Input] { 118 | return items.map { deprocess1(item: $0) } 119 | } 120 | } 121 | 122 | //cell42 123 | public struct NoopProcessor: Processor { 124 | public init() {} 125 | 126 | public mutating func initState(items: [Item]) {} 127 | 128 | public func process1 (item: Item) -> Item { return item } 129 | public func deprocess1(item: Item) -> Item { return item } 130 | } 131 | 132 | //cell43 133 | public struct CategoryProcessor: Processor { 134 | public init() {} 135 | public var vocab: [String]? = nil 136 | public var reverseMap: [String: Int32]? = nil 137 | 138 | public mutating func initState(items: [String]) { 139 | vocab = Array(Set(items)).sorted() 140 | reverseMap = [:] 141 | for (i,x) in vocab!.enumerated() { reverseMap![x] = Int32(i) } 142 | } 143 | 144 | public func process1 (item: String) -> Int32 { return reverseMap![item]! } 145 | public func deprocess1(item: Int32) -> String { return vocab![Int(item)] } 146 | } 147 | 148 | //cell46 149 | public struct LabeledItemList where PI: Processor, PL: Processor{ 150 | public var items: [PI.Output] 151 | public var labels: [PL.Output] 152 | public let path: Path 153 | public var procItem: PI 154 | public var procLabel: PL 155 | 156 | public init(rawItems: [PI.Input], rawLabels: [PL.Input], path: Path, procItem: PI, procLabel: PL){ 157 | (self.procItem,self.procLabel,self.path) = (procItem,procLabel,path) 158 | self.items = procItem.process(items: rawItems) 159 | self.labels = procLabel.process(items: rawLabels) 160 | } 161 | 162 | public init(_ il: ItemList, fromFunc: (PI.Input) -> PL.Input, procItem: PI, procLabel: PL){ 163 | self.init(rawItems: il.items, 164 | rawLabels: il.items.map{ fromFunc($0)}, 165 | path: il.path, 166 | procItem: procItem, 167 | procLabel: procLabel) 168 | } 169 | 170 | public func rawItem (_ idx: Int) -> PI.Input { return procItem.deprocess1 (item: items[idx]) } 171 | public func rawLabel(_ idx: Int) -> PL.Input { return procLabel.deprocess1(item: labels[idx]) } 172 | } 173 | 174 | //cell47 175 | public struct SplitLabeledData where PI: Processor, PL: Processor{ 176 | public let train: LabeledItemList 177 | public let valid: LabeledItemList 178 | public var path: Path { return train.path } 179 | 180 | public init(train: LabeledItemList, valid: LabeledItemList){ 181 | (self.train, self.valid) = (train, valid) 182 | } 183 | 184 | public init(_ sd: SplitData, fromFunc: (PI.Input) -> PL.Input, procItem: inout PI, procLabel: inout PL){ 185 | procItem.initState(items: sd.train.items) 186 | let trainLabels = sd.train.items.map{ fromFunc($0) } 187 | procLabel.initState(items: trainLabels) 188 | self.init(train: LabeledItemList(rawItems: sd.train.items, rawLabels: trainLabels, path: sd.path, 189 | procItem: procItem, procLabel: procLabel), 190 | valid: LabeledItemList(sd.valid, fromFunc: fromFunc, procItem: procItem, procLabel: procLabel)) 191 | } 192 | } 193 | 194 | /// Make a labeled data without an input processor, by defaulting to a noop processor. 195 | public func makeLabeledData(_ sd: SplitData, fromFunc: (T) -> PL.Input, procLabel: inout PL) 196 | -> SplitLabeledData, PL> { 197 | var pi = NoopProcessor() 198 | return SplitLabeledData(sd, fromFunc: fromFunc, procItem: &pi, procLabel: &procLabel) 199 | } 200 | 201 | 202 | //cell48 203 | public func parentLabeler(_ fName: Path) -> String { return fName.parent.basename() } 204 | 205 | //cell53 206 | public struct LabeledElement: TensorGroup { 207 | public var xb: I 208 | public var yb: L 209 | 210 | public init(xb: I, yb: L) { 211 | (self.xb, self.yb) = (xb, yb) 212 | } 213 | 214 | // Explicit implementation to make this struct work well with LazyTensor. 215 | // These will be derived automatically in the future. 216 | public var _tensorHandles: [_AnyTensorHandle] { 217 | xb._tensorHandles + yb._tensorHandles 218 | } 219 | 220 | public init( 221 | _handles: C 222 | ) where C.Element: _AnyTensorHandle { 223 | let xStart = _handles.startIndex 224 | let xEnd = _handles.index( 225 | xStart, offsetBy: Int(I._tensorHandleCount)) 226 | self.xb = I.init(_handles: _handles[xStart.. ( 234 | itemToTensor: ([PI.Output]) -> XB, labelToTensor: ([PL.Output]) -> YB, bs: Int = 64 235 | ) -> DataBunch> where XB: TensorGroup, YB: TensorGroup { 236 | let trainDs = Dataset>( 237 | elements: LabeledElement(xb: itemToTensor(train.items), yb: labelToTensor(train.labels))) 238 | let validDs = Dataset>( 239 | elements: LabeledElement(xb: itemToTensor(valid.items), yb: labelToTensor(valid.labels))) 240 | return DataBunch(train: trainDs, valid: validDs, 241 | trainLen: train.items.count, validLen: valid.items.count, 242 | bs: bs) 243 | } 244 | } 245 | 246 | //cell55 247 | public func pathsToTensor(_ paths: [Path]) -> StringTensor { return StringTensor(paths.map{ $0.string })} 248 | public func intsToTensor(_ items: [Int32]) -> Tensor { return Tensor(items)} 249 | 250 | //cell59 251 | public func transformData( 252 | _ data: DataBunch>, 253 | nWorkers:Int=1, 254 | tfmItem: (I) -> TI 255 | ) -> DataBunch> 256 | where I: TensorGroup, TI: TensorGroup & Differentiable, L: TensorGroup{ 257 | return DataBunch(train: data.train.innerDs.map(parallelCallCount: nWorkers){ DataBatch(xb: tfmItem($0.xb), yb: $0.yb) }, 258 | valid: data.valid.innerDs.map(parallelCallCount: nWorkers){ DataBatch(xb: tfmItem($0.xb), yb: $0.yb) }, 259 | trainLen: data.train.dsCount, 260 | validLen: data.valid.dsCount, 261 | bs: data.train.bs) 262 | } 263 | 264 | //cell60 265 | public func openAndResize(fname: StringTensor, size: Int) -> TF{ 266 | let decodedImg = StringTensor(readFile: fname).decodeJpeg(channels: 3) 267 | let resizedImg = Tensor(_Raw.resizeBilinear( 268 | images: Tensor([decodedImg]), 269 | size: Tensor([Int32(size), Int32(size)]))) / 255.0 270 | return resizedImg[0] 271 | } 272 | 273 | //cell62 274 | public extension FADataset { 275 | func oneBatch() -> Element? { 276 | for batch in ds { return batch } 277 | return nil 278 | } 279 | } 280 | 281 | //cell64 282 | public func showImages(_ xb: TF, labels: [String]? = nil) { 283 | let (rows,cols) = (3,3) 284 | plt.figure(figsize: [9, 9]) 285 | for i in 0..<(rows * cols) { 286 | let img = plt.subplot(rows, cols, i + 1) 287 | img.axis("off") 288 | let x = xb[i].makeNumpyArray() 289 | img.imshow(x) 290 | if labels != nil { img.set_title(labels![i]) } 291 | if (i + 1) >= (rows * cols) { break } 292 | } 293 | plt.show() 294 | } 295 | 296 | //cell74 297 | public let imagenetStats = (mean: TF([0.485, 0.456, 0.406]), std: TF([0.229, 0.224, 0.225])) 298 | 299 | //cell75 300 | public func prevPow2(_ x: Int) -> Int { 301 | var res = 1 302 | while res <= x { res *= 2 } 303 | return res / 2 304 | } 305 | 306 | //cell76 307 | public struct CNNModel: Layer { 308 | public var convs: [ConvBN] 309 | public var pool = FAGlobalAvgPool2D() 310 | public var linear: FADense 311 | 312 | public init(channelIn: Int, nOut: Int, filters: [Int]){ 313 | convs = [] 314 | let (l1,l2) = (channelIn, prevPow2(channelIn * 9)) 315 | convs = [ConvBN(l1, l2, stride: 1), 316 | ConvBN(l2, l2*2, stride: 2), 317 | ConvBN(l2*2, l2*4, stride: 2)] 318 | let allFilters = [l2*4] + filters 319 | for i in 0..(filters.last!, nOut) 321 | } 322 | 323 | @differentiable 324 | public func callAsFunction(_ input: TF) -> TF { 325 | return linear(pool(convs(input))) 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Callbacks.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 04_callbacks.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | 15 | //cell8 16 | public struct BasicModel: Layer { 17 | public var layer1, layer2: FADense 18 | 19 | public init(nIn: Int, nHid: Int, nOut: Int){ 20 | layer1 = FADense(nIn, nHid, activation: relu) 21 | layer2 = FADense(nHid, nOut) 22 | } 23 | 24 | @differentiable 25 | public func callAsFunction(_ input: Tensor) -> Tensor { 26 | return layer2(layer1(input)) 27 | } 28 | } 29 | 30 | //cell13 31 | public struct FADataset where Element: TensorGroup { 32 | public var innerDs: Dataset 33 | public var shuffle = false 34 | public var bs = 64 35 | public var dsCount: Int 36 | 37 | public var count: Int { 38 | return dsCount%bs == 0 ? dsCount/bs : dsCount/bs+1 39 | } 40 | 41 | public var ds: Dataset { 42 | if !shuffle { return innerDs.batched(bs)} 43 | let seed = Int64.random(in: Int64.min.., len: Int, shuffle: Bool = false, bs: Int = 64) { 48 | (self.innerDs,self.dsCount,self.shuffle,self.bs) = (ds, len, shuffle, bs) 49 | } 50 | } 51 | 52 | //cell15 53 | public struct DataBunch where Element: TensorGroup{ 54 | public var train, valid: FADataset 55 | 56 | public init(train: Dataset, valid: Dataset, trainLen: Int, validLen: Int, bs: Int = 64) { 57 | self.train = FADataset(train, len: trainLen, shuffle: true, bs: bs) 58 | self.valid = FADataset(valid, len: validLen, shuffle: false, bs: 2*bs) 59 | } 60 | } 61 | 62 | //cell17 63 | public func mnistDataBunch(path: Path = mnistPath, flat: Bool = false, bs: Int = 64) 64 | -> DataBunch> { 65 | let (xTrain,yTrain,xValid,yValid) = loadMNIST(path: path, flat: flat) 66 | return DataBunch(train: Dataset(elements: DataBatch(xb:xTrain, yb: yTrain)), 67 | valid: Dataset(elements: DataBatch(xb:xValid, yb: yValid)), 68 | trainLen: xTrain.shape[0], 69 | validLen: xValid.shape[0], 70 | bs: bs) 71 | } 72 | 73 | //cell22 74 | public extension Sequence { 75 | func first() -> Element? { 76 | return first(where: {_ in true}) 77 | } 78 | } 79 | 80 | //cell29 81 | public enum LearnerAction: Error { 82 | case skipEpoch(reason: String) 83 | case skipBatch(reason: String) 84 | case stop(reason: String) 85 | } 86 | 87 | //cell32 88 | /// Initializes and trains a model on a given dataset. 89 | public final class Learner 91 | where Opt.Scalar: Differentiable, 92 | Opt.Model: Layer, 93 | // Constrain model input to Tensor, to work around 94 | // https://forums.fast.ai/t/fix-ad-crash-in-learner/42970. 95 | Opt.Model.Input == Tensor 96 | { 97 | public typealias Model = Opt.Model 98 | public typealias Input = Model.Input 99 | public typealias Output = Model.Output 100 | public typealias Data = DataBunch> 101 | public typealias Loss = TF 102 | public typealias Optimizer = Opt 103 | public typealias EventHandler = (Learner) throws -> Void 104 | 105 | /// A wrapper class to hold the loss function, to work around 106 | // https://forums.fast.ai/t/fix-ad-crash-in-learner/42970. 107 | public final class LossFunction { 108 | public typealias F = @differentiable (Model.Output, @noDerivative Label) -> Loss 109 | public var f: F 110 | init(_ f: @escaping F) { self.f = f } 111 | } 112 | 113 | public var data: Data 114 | public var opt: Optimizer 115 | public var lossFunc: LossFunction 116 | public var model: Model 117 | 118 | public var currentInput: Input! 119 | public var currentTarget: Label! 120 | public var currentOutput: Output! 121 | 122 | public private(set) var epochCount = 0 123 | public private(set) var currentEpoch = 0 124 | public private(set) var currentGradient = Model.TangentVector.zero 125 | public private(set) var currentLoss = Loss.zero 126 | public private(set) var inTrain = false 127 | public private(set) var pctEpochs = Float.zero 128 | public private(set) var currentIter = 0 129 | public private(set) var iterCount = 0 130 | 131 | open class Delegate { 132 | open var order: Int { return 0 } 133 | public init () {} 134 | 135 | open func trainingWillStart(learner: Learner) throws {} 136 | open func trainingDidFinish(learner: Learner) throws {} 137 | open func epochWillStart(learner: Learner) throws {} 138 | open func epochDidFinish(learner: Learner) throws {} 139 | open func validationWillStart(learner: Learner) throws {} 140 | open func batchWillStart(learner: Learner) throws {} 141 | open func batchDidFinish(learner: Learner) throws {} 142 | open func didProduceNewGradient(learner: Learner) throws {} 143 | open func optimizerDidUpdate(learner: Learner) throws {} 144 | open func batchSkipped(learner: Learner, reason:String) throws {} 145 | open func epochSkipped(learner: Learner, reason:String) throws {} 146 | open func trainingStopped(learner: Learner, reason:String) throws {} 147 | /// 148 | /// TODO: learnerDidProduceNewOutput and learnerDidProduceNewLoss need to 149 | /// be differentiable once we can have the loss function inside the Learner 150 | } 151 | 152 | public var delegates: [Delegate] = [] { 153 | didSet { delegates.sort { $0.order < $1.order } } 154 | } 155 | 156 | public init(data: Data, lossFunc: @escaping LossFunction.F, 157 | optFunc: (Model) -> Optimizer, modelInit: ()->Model) { 158 | (self.data,self.lossFunc) = (data,LossFunction(lossFunc)) 159 | model = modelInit() 160 | opt = optFunc(self.model) 161 | } 162 | } 163 | 164 | //cell34 165 | extension Learner { 166 | private func evaluate(onBatch batch: DataBatch) throws { 167 | currentOutput = model(currentInput) 168 | currentLoss = lossFunc.f(currentOutput, currentTarget) 169 | } 170 | 171 | private func train(onBatch batch: DataBatch) throws { 172 | let (xb,yb) = (currentInput!,currentTarget!) //We still have to force-unwrap those for AD... 173 | (currentLoss, currentGradient) = valueWithGradient(at: model) { model -> Loss in 174 | let y = model(xb) 175 | self.currentOutput = y 176 | return self.lossFunc.f(y, yb) 177 | } 178 | for d in delegates { try d.didProduceNewGradient(learner: self) } 179 | opt.update(&model, along: self.currentGradient) 180 | } 181 | 182 | private func train(onDataset ds: FADataset>) throws { 183 | iterCount = ds.count 184 | for batch in ds.ds { 185 | (currentInput, currentTarget) = (batch.xb, batch.yb) 186 | do { 187 | for d in delegates { try d.batchWillStart(learner: self) } 188 | if inTrain { try train(onBatch: batch) } else { try evaluate(onBatch: batch) } 189 | } 190 | catch LearnerAction.skipBatch(let reason) { 191 | for d in delegates {try d.batchSkipped(learner: self, reason:reason)} 192 | } 193 | for d in delegates { try d.batchDidFinish(learner: self) } 194 | } 195 | } 196 | } 197 | 198 | //cell36 199 | extension Learner { 200 | /// Starts fitting. 201 | /// - Parameter epochCount: The number of epochs that will be run. 202 | public func fit(_ epochCount: Int) throws { 203 | self.epochCount = epochCount 204 | do { 205 | for d in delegates { try d.trainingWillStart(learner: self) } 206 | for i in 0.. TrainEvalDelegate { return TrainEvalDelegate() } 257 | } 258 | 259 | //cell52 260 | extension Learner { 261 | public class AvgMetric: Delegate { 262 | public let metrics: [(Output, Label) -> TF] 263 | var total: Int = 0 264 | var partials = [TF]() 265 | 266 | public init(metrics: [(Output, Label) -> TF]) { self.metrics = metrics} 267 | 268 | public override func epochWillStart(learner: Learner) { 269 | total = 0 270 | partials = Array(repeating: Tensor(0), count: metrics.count + 1) 271 | } 272 | 273 | public override func batchDidFinish(learner: Learner) { 274 | if !learner.inTrain{ 275 | let bs = learner.currentInput!.shape[0] //Possible because Input is TF for now 276 | total += bs 277 | partials[0] += Float(bs) * learner.currentLoss 278 | for i in 1...metrics.count{ 279 | partials[i] += Float(bs) * metrics[i-1](learner.currentOutput!, learner.currentTarget!) 280 | } 281 | } 282 | } 283 | 284 | public override func epochDidFinish(learner: Learner) { 285 | for i in 0...metrics.count {partials[i] = partials[i] / Float(total)} 286 | print("Epoch \(learner.currentEpoch): \(partials)") 287 | } 288 | } 289 | 290 | public func makeAvgMetric(metrics: [(Output, Label) -> TF]) -> AvgMetric{ 291 | return AvgMetric(metrics: metrics) 292 | } 293 | } 294 | 295 | //cell57 296 | extension Learner { 297 | public class Normalize: Delegate { 298 | public let mean, std: TF 299 | public init(mean: TF, std: TF) { (self.mean,self.std) = (mean,std) } 300 | 301 | public override func batchWillStart(learner: Learner) { 302 | learner.currentInput = (learner.currentInput! - mean) / std 303 | } 304 | } 305 | 306 | public func makeNormalize(mean: TF, std: TF) -> Normalize{ 307 | return Normalize(mean: mean, std: std) 308 | } 309 | } 310 | 311 | //cell59 312 | public let mnistStats = (mean: TF(0.13066047), std: TF(0.3081079)) 313 | -------------------------------------------------------------------------------- /Sources/SwiftAI/FastaiLayers.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 01a_fastai_layers.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell2 12 | import Path 13 | import TensorFlow 14 | 15 | //cell6 16 | public extension Tensor where Scalar: TensorFlowFloatingPoint { 17 | init(kaimingNormal shape: TensorShape, negativeSlope: Double = 1.0) { 18 | // Assumes Leaky ReLU nonlinearity 19 | let gain = Scalar.init(TensorFlow.sqrt(2.0 / (1.0 + TensorFlow.pow(negativeSlope, 2)))) 20 | let spatialDimCount = shape.count - 2 21 | let receptiveField = shape[0..(gain/TensorFlow.sqrt(Scalar(fanIn))) 25 | } 26 | } 27 | 28 | //cell8 29 | public extension Tensor where Scalar: TensorFlowFloatingPoint { 30 | func std() -> Tensor { return standardDeviation() } 31 | func std(alongAxes a: [Int]) -> Tensor { return standardDeviation(alongAxes: a) } 32 | func std(alongAxes a: Tensor) -> Tensor { return standardDeviation(alongAxes: a) } 33 | func std(alongAxes a: Int...) -> Tensor { return standardDeviation(alongAxes: a) } 34 | func std(squeezingAxes a: [Int]) -> Tensor { return standardDeviation(squeezingAxes: a) } 35 | func std(squeezingAxes a: Tensor) -> Tensor { return standardDeviation(squeezingAxes: a) } 36 | func std(squeezingAxes a: Int...) -> Tensor { return standardDeviation(squeezingAxes: a) } 37 | } 38 | 39 | //cell13 40 | 41 | // FALayer is a layer that supports callbacks through its LayerDelegate. 42 | public protocol FALayer: Layer { 43 | var delegates: [(Output) -> ()] { get set } 44 | 45 | // FALayer's will implement this instead of `func call`. 46 | @differentiable 47 | func forward(_ input: Input) -> Output 48 | 49 | associatedtype Input 50 | associatedtype Output 51 | } 52 | 53 | //cell15 54 | public extension FALayer { 55 | @differentiable 56 | @differentiable(wrt: (self)) 57 | func callAsFunction(_ input: Input) -> Output { 58 | let activation = forward(input) 59 | for d in delegates { d(activation) } 60 | return activation 61 | } 62 | 63 | mutating func addDelegate(_ d: @escaping (Output) -> ()) { delegates.append(d) } 64 | } 65 | 66 | 67 | //cell19 68 | @frozen 69 | public struct FADense: FALayer { 70 | // Note: remove the explicit typealiases after TF-603 is resolved. 71 | public typealias Input = Tensor 72 | public typealias Output = Tensor 73 | public var weight: Tensor 74 | public var bias: Tensor 75 | public typealias Activation = @differentiable (Tensor) -> Tensor 76 | @noDerivative public var delegates: [(Output) -> ()] = [] 77 | @noDerivative public let activation: Activation 78 | 79 | public init( 80 | weight: Tensor, 81 | bias: Tensor, 82 | activation: @escaping Activation 83 | ) { 84 | self.weight = weight 85 | self.bias = bias 86 | self.activation = activation 87 | } 88 | 89 | @differentiable 90 | public func forward(_ input: Tensor) -> Tensor { 91 | return activation(input • weight + bias) 92 | } 93 | } 94 | 95 | public extension FADense { 96 | init(_ nIn: Int, _ nOut: Int, activation: @escaping Activation = identity) { 97 | self.init(weight: Tensor(kaimingNormal: [nIn, nOut], negativeSlope: 1.0), 98 | bias: Tensor(zeros: [nOut]), 99 | activation: activation) 100 | } 101 | } 102 | 103 | //cell21 104 | 105 | @frozen 106 | public struct FANoBiasConv2D: FALayer { 107 | // TF-603 workaround. 108 | public typealias Input = Tensor 109 | public typealias Output = Tensor 110 | 111 | public var filter: Tensor 112 | public typealias Activation = @differentiable (Tensor) -> Tensor 113 | @noDerivative public let activation: Activation 114 | @noDerivative public let strides: (Int, Int) 115 | @noDerivative public let padding: Padding 116 | @noDerivative public var delegates: [(Output) -> ()] = [] 117 | 118 | public init( 119 | filter: Tensor, 120 | activation: @escaping Activation, 121 | strides: (Int, Int), 122 | padding: Padding 123 | ) { 124 | self.filter = filter 125 | self.activation = activation 126 | self.strides = strides 127 | self.padding = padding 128 | } 129 | 130 | @differentiable 131 | public func forward(_ input: Tensor) -> Tensor { 132 | return activation(conv2D(input, filter: filter, 133 | strides: (1, strides.0, strides.1, 1), 134 | padding: padding)) 135 | } 136 | } 137 | 138 | public extension FANoBiasConv2D { 139 | init( 140 | filterShape: (Int, Int, Int, Int), 141 | strides: (Int, Int) = (1, 1), 142 | padding: Padding = .same, 143 | activation: @escaping Activation = identity 144 | ) { 145 | let filterTensorShape = TensorShape([ 146 | filterShape.0, filterShape.1, 147 | filterShape.2, filterShape.3]) 148 | self.init( 149 | filter: Tensor(kaimingNormal: filterTensorShape, negativeSlope: 1.0), 150 | activation: activation, 151 | strides: strides, 152 | padding: padding) 153 | } 154 | } 155 | 156 | public extension FANoBiasConv2D { 157 | init(_ cIn: Int, _ cOut: Int, ks: Int, stride: Int = 1, padding: Padding = .same, 158 | activation: @escaping Activation = identity){ 159 | self.init(filterShape: (ks, ks, cIn, cOut), 160 | strides: (stride, stride), 161 | padding: padding, 162 | activation: activation) 163 | } 164 | } 165 | 166 | //cell22 167 | 168 | @frozen 169 | public struct FAConv2D: FALayer { 170 | // Note: remove the explicit typealiases after TF-603 is resolved. 171 | public typealias Input = Tensor 172 | public typealias Output = Tensor 173 | 174 | public var filter: Tensor 175 | public var bias: Tensor 176 | public typealias Activation = @differentiable (Tensor) -> Tensor 177 | @noDerivative public let activation: Activation 178 | @noDerivative public let strides: (Int, Int) 179 | @noDerivative public let padding: Padding 180 | @noDerivative public var delegates: [(Output) -> ()] = [] 181 | 182 | public init( 183 | filter: Tensor, 184 | bias: Tensor, 185 | activation: @escaping Activation, 186 | strides: (Int, Int), 187 | padding: Padding 188 | ) { 189 | self.filter = filter 190 | self.bias = bias 191 | self.activation = activation 192 | self.strides = strides 193 | self.padding = padding 194 | } 195 | 196 | @differentiable 197 | public func forward(_ input: Tensor) -> Tensor { 198 | return activation(conv2D(input, filter: filter, 199 | strides: (1, strides.0, strides.1, 1), 200 | padding: padding) + bias) 201 | } 202 | } 203 | 204 | public extension FAConv2D { 205 | init( 206 | filterShape: (Int, Int, Int, Int), 207 | strides: (Int, Int) = (1, 1), 208 | padding: Padding = .same, 209 | activation: @escaping Activation = identity 210 | ) { 211 | let filterTensorShape = TensorShape([ 212 | filterShape.0, filterShape.1, 213 | filterShape.2, filterShape.3]) 214 | self.init( 215 | filter: Tensor(kaimingNormal: filterTensorShape, negativeSlope: 1.0), 216 | bias: Tensor(zeros: TensorShape([filterShape.3])), 217 | activation: activation, 218 | strides: strides, 219 | padding: padding) 220 | } 221 | } 222 | 223 | public extension FAConv2D { 224 | init(_ cIn: Int, _ cOut: Int, ks: Int, stride: Int = 1, padding: Padding = .same, 225 | activation: @escaping Activation = identity){ 226 | self.init(filterShape: (ks, ks, cIn, cOut), 227 | strides: (stride, stride), 228 | padding: padding, 229 | activation: activation) 230 | } 231 | } 232 | 233 | //cell24 234 | 235 | @frozen 236 | public struct FAAvgPool2D: FALayer,ParameterlessLayer { 237 | // TF-603 workaround. 238 | public typealias Input = Tensor 239 | public typealias Output = Tensor 240 | 241 | @noDerivative let poolSize: (Int, Int, Int, Int) 242 | @noDerivative let strides: (Int, Int, Int, Int) 243 | @noDerivative let padding: Padding 244 | @noDerivative public var delegates: [(Output) -> ()] = [] 245 | 246 | public init( 247 | poolSize: (Int, Int, Int, Int), 248 | strides: (Int, Int, Int, Int), 249 | padding: Padding 250 | ) { 251 | self.poolSize = poolSize 252 | self.strides = strides 253 | self.padding = padding 254 | } 255 | 256 | public init(poolSize: (Int, Int), strides: (Int, Int), padding: Padding = .valid) { 257 | self.poolSize = (1, poolSize.0, poolSize.1, 1) 258 | self.strides = (1, strides.0, strides.1, 1) 259 | self.padding = padding 260 | } 261 | 262 | public init(_ sz: Int, padding: Padding = .valid) { 263 | poolSize = (1, sz, sz, 1) 264 | strides = (1, sz, sz, 1) 265 | self.padding = padding 266 | } 267 | 268 | @differentiable 269 | public func forward(_ input: Tensor) -> Tensor { 270 | return avgPool2D(input, filterSize: poolSize, strides: strides, padding: padding) 271 | } 272 | } 273 | 274 | //cell25 275 | 276 | @frozen 277 | public struct FAGlobalAvgPool2D: FALayer,ParameterlessLayer { 278 | // TF-603 workaround. 279 | public typealias Input = Tensor 280 | public typealias Output = Tensor 281 | @noDerivative public var delegates: [(Output) -> ()] = [] 282 | 283 | public init() {} 284 | 285 | @differentiable 286 | public func forward(_ input: Tensor) -> Tensor { 287 | return input.mean(squeezingAxes: [1,2]) 288 | } 289 | } 290 | 291 | //cell27 292 | extension Array: Module where Element: Layer, Element.Input == Element.Output { 293 | public typealias Input = Element.Input 294 | public typealias Output = Element.Output 295 | 296 | @differentiable(wrt: (self, input)) 297 | public func callAsFunction(_ input: Input) -> Output { 298 | return self.differentiableReduce(input) { $1($0) } 299 | } 300 | } 301 | extension Array: Layer where Element: Layer, Element.Input == Element.Output {} 302 | 303 | //cell29 304 | extension KeyPathIterable { 305 | public var keyPaths: [WritableKeyPath>] { 306 | return recursivelyAllWritableKeyPaths(to: Tensor.self) 307 | } 308 | } 309 | 310 | //cell31 311 | public func ** (lhs: Int, rhs: Int) -> Int { 312 | return Int(pow(Double(lhs), Double(rhs))) 313 | } 314 | 315 | public func ** (lhs: Double, rhs: Double) -> Double { 316 | return pow(lhs, rhs) 317 | } 318 | 319 | public func **(_ x: T, _ y: T) -> T { 320 | return T(pow(Double(x), Double(y))) 321 | } 322 | 323 | public func **(_ x: Tensor, _ y: Tensor) -> Tensor 324 | where T : TensorFlowFloatingPoint { return pow(x, y)} 325 | 326 | public func **(_ x: T, _ y: Tensor) -> Tensor 327 | where T : TensorFlowFloatingPoint { return pow(x, y)} 328 | 329 | public func **(_ x: Tensor, _ y: T) -> Tensor 330 | where T : TensorFlowFloatingPoint { return pow(x, y)} 331 | 332 | //cell33 333 | public extension Differentiable { 334 | @differentiable 335 | func compose(_ l1: L1, _ l2: L2) -> L2.Output 336 | where L1.Input == Self, L1.Output == L2.Input { 337 | return sequenced(through: l1, l2) 338 | } 339 | 340 | @differentiable 341 | func compose(_ l1: L1, _ l2: L2, _ l3: L3) -> L3.Output 342 | where L1.Input == Self, L1.Output == L2.Input, L2.Output == L3.Input { 343 | return sequenced(through: l1, l2, l3) 344 | } 345 | 346 | @differentiable 347 | func compose( 348 | _ l1: L1, _ l2: L2, _ l3: L3, _ l4: L4 349 | ) -> L4.Output 350 | where L1.Input == Self, L1.Output == L2.Input, L2.Output == L3.Input, 351 | L3.Output == L4.Input { 352 | return sequenced(through: l1, l2, l3, l4) 353 | } 354 | 355 | @differentiable 356 | func compose( 357 | _ l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5 358 | ) -> L5.Output 359 | where L1.Input == Self, L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, 360 | L4.Output == L5.Input { 361 | return sequenced(through: l1, l2, l3, l4, l5) 362 | } 363 | 364 | @differentiable 365 | func compose( 366 | _ l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5, _ l6: L6 367 | ) -> L6.Output 368 | where L1.Input == Self, L1.Output == L2.Input, L2.Output == L3.Input, L3.Output == L4.Input, 369 | L4.Output == L5.Input, L5.Output == L6.Input { 370 | return sequenced(through: l1, l2, l3, l4, l5, l6) 371 | } 372 | } 373 | -------------------------------------------------------------------------------- /tools/export_import.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Installing packages:\n", 13 | "\t.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")\n", 14 | "\t\tPath\n", 15 | "With SwiftPM flags: []\n", 16 | "Working in: /tmp/tmpml0z_fjz/swift-install\n", 17 | "/home/sgugger/swift/usr/bin/swift-build: /home/sgugger/anaconda3/lib/libcurl.so.4: no version information available (required by /home/sgugger/swift/usr/lib/swift/linux/libFoundationNetworking.so)\n", 18 | "[1/10] Compiling Path Path+StringConvertibles.swift\n", 19 | "[2/10] Compiling Path Path+CommonDirectories.swift\n", 20 | "[3/10] Compiling Path Path->Bool.swift\n", 21 | "[4/10] Compiling Path Path+Attributes.swift\n", 22 | "[5/10] Compiling Path Path+ls.swift\n", 23 | "[6/10] Compiling Path Extensions.swift\n", 24 | "[7/10] Compiling Path Path+Codable.swift\n", 25 | "[8/10] Compiling Path Path+FileManager.swift\n", 26 | "[9/10] Compiling Path Path.swift\n", 27 | "[10/11] Merging module Path\n", 28 | "[11/14] Wrapping AST for Path for debugging\n", 29 | "[12/14] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", 30 | "[13/15] Merging module jupyterInstalledPackages\n", 31 | "[14/15] Wrapping AST for jupyterInstalledPackages for debugging\n", 32 | "[15/15] Linking libjupyterInstalledPackages.so\n", 33 | "Initializing Swift...\n", 34 | "Installation complete!\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "%install-location $cwd/swift-install\n", 40 | "%install '.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")' Path" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "import Foundation\n", 50 | "import Path" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 3, 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "/home/sgugger/git/swiftai\r\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "let path = Path.cwd.parent\n", 68 | "print(path)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 4, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "public extension String {\n", 78 | " func findFirst(pat: String) -> Range? {\n", 79 | " return range(of: pat, options: .regularExpression)\n", 80 | " }\n", 81 | " func hasMatch(pat: String) -> Bool {\n", 82 | " return findFirst(pat:pat) != nil\n", 83 | " }\n", 84 | " func withMaj() -> String {\n", 85 | " return prefix(1).capitalized + dropFirst()\n", 86 | " }\n", 87 | "}" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "## Notebook to script" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 5, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "public func nbNameToScriptName(_ name: String) -> String {\n", 104 | " var splits = name.components(separatedBy: \"_\")\n", 105 | " splits = splits[1...].map { $0.withMaj() }\n", 106 | " return splits.joined(separator: \"\")\n", 107 | "}" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 6, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "public func readNb(_ fname: Path) -> [String:Any] {\n", 117 | " let data = try! Data(contentsOf: fname.url)\n", 118 | " return try! JSONSerialization.jsonObject(with: data, options: .allowFragments) as! [String: Any]\n", 119 | "}" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 7, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "public func notebookToScript(_ fname: Path, dest: Path?=nil) {\n", 129 | " let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n", 130 | " let destFname = (dest ?? fname.parent) / newName\n", 131 | " let cells = readNb(fname)[\"cells\"] as! [[String:Any]]\n", 132 | " var module = \"\"\"\n", 133 | "/*\n", 134 | "This file was autogenerated from \\(fname.basename())\n", 135 | " \n", 136 | "If you edit it, be sure that:\n", 137 | " 1. there is no diff between this file and the corresponding notebook prior to editing\n", 138 | " 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook\n", 139 | " \n", 140 | "Run *** when you are done to update the notebooks with your change.\n", 141 | "*/\n", 142 | " \n", 143 | "\"\"\"\n", 144 | " for (i,cell) in cells.enumerated() {\n", 145 | " if let source = cell[\"source\"] as? [String], !source.isEmpty, source[0].hasMatch(pat: #\"^\\s*//\\s*export\\s*$\"#) {\n", 146 | " module.append(\"\\n//cell\\(i)\\n\\(source[1...].joined())\\n\")\n", 147 | " }\n", 148 | " }\n", 149 | " try! module.write(to: destFname, encoding: .utf8)\n", 150 | "}" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 8, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "let fname = path/\"/nbs/00_load_data.ipynb\"\n", 160 | "let dest = path/\"Sources/SwiftAI\"" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 9, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "notebookToScript(fname, dest: dest)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 10, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "public func makeLibrary(_ nbFolder: Path, dest: Path?=nil){\n", 179 | " for entry in try! nbFolder.ls() where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\"^\\d+[a-z]*_.*ipynb$\"#) {\n", 180 | " print(\"Converting \\(entry.path.basename())\")\n", 181 | " notebookToScript(entry.path, dest: dest)\n", 182 | " }\n", 183 | "}" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 11, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "let nbFolder = path/\"nbs\"" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 12, 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "name": "stdout", 202 | "output_type": "stream", 203 | "text": [ 204 | "Converting 02a_why_sqrt5.ipynb\n", 205 | "Converting 10_mixup_ls.ipynb\n", 206 | "Converting 05_anneal.ipynb\n", 207 | "Converting 09_optimizer.ipynb\n", 208 | "Converting 04_callbacks.ipynb\n", 209 | "Converting 08_data_block.ipynb\n", 210 | "Converting 08a_heterogeneous_dictionary.ipynb\n", 211 | "Converting 06_cuda.ipynb\n", 212 | "Converting 01_matmul.ipynb\n", 213 | "Converting 02_fully_connected.ipynb\n", 214 | "Converting 01a_fastai_layers.ipynb\n", 215 | "Converting 11_imagenette.ipynb\n", 216 | "Converting 05b_early_stopping.ipynb\n", 217 | "Converting 00_load_data.ipynb\n", 218 | "Converting 03_minibatch_training.ipynb\n", 219 | "Converting 07_batchnorm.ipynb\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "makeLibrary(nbFolder, dest:dest)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "## Script to notebook" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 14, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "let fname = dest/\"LoadData.swift\"" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": 15, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "public func readScript(_ fname: Path) -> String {\n", 250 | " let data = try! Data(contentsOf: fname.url)\n", 251 | " return String(data: data, encoding: .utf8)!\n", 252 | "}" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 16, 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "public func writeNotebook(_ nbFname: Path, nbData: [String: Any]) {\n", 262 | " let outData = try! JSONSerialization.data(withJSONObject: nbData, options: .prettyPrinted) \n", 263 | " let jsonString = String(data: outData, encoding: .utf8)! \n", 264 | " do { try jsonString.write(to: nbFname, encoding: .utf8) }\n", 265 | " catch { \"Couldn't save notebook\" }\n", 266 | "}" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 17, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "public func scriptToNotebook(_ fname: Path, nbFolder: Path){\n", 276 | " let code = readScript(fname)\n", 277 | " let codeCells = code.components(separatedBy: \"//cell\")\n", 278 | " let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n", 279 | " let nbFname = nbFolder / nbName\n", 280 | " var jsonData = readNb(nbFname)\n", 281 | " var cells = jsonData[\"cells\"] as! [[String:Any]]\n", 282 | " for c in codeCells[1...] {\n", 283 | " var lines = c.components(separatedBy: \"\\n\")\n", 284 | " let idx: Int = Int(lines[0])!\n", 285 | " var i = lines.count-1\n", 286 | " while lines[i].isEmpty { i -= 1}\n", 287 | " if i > 1 {\n", 288 | " for i in 1...(i-1) { lines[i].append(\"\\n\") }\n", 289 | " }\n", 290 | " lines[0] = \"// export\\n\"\n", 291 | " cells[idx][\"source\"] = Array(lines[...i])\n", 292 | " }\n", 293 | " jsonData[\"cells\"] = cells\n", 294 | " writeNotebook(nbFname, nbData: jsonData)\n", 295 | "}" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": 19, 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [ 304 | "scriptToNotebook(fname, nbFolder: nbFolder)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": 18, 310 | "metadata": {}, 311 | "outputs": [], 312 | "source": [ 313 | "public func updateNotebooks(_ scriptsFolder: Path, nbFolder: Path) {\n", 314 | " for entry in try! scriptsFolder.ls() where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\".swift$\"#) {\n", 315 | " print(\"Updating nb from \\(entry.path.basename())\")\n", 316 | " scriptToNotebook(entry.path, nbFolder: nbFolder)\n", 317 | " }\n", 318 | "}" 319 | ] 320 | }, 321 | { 322 | "cell_type": "code", 323 | "execution_count": 20, 324 | "metadata": {}, 325 | "outputs": [ 326 | { 327 | "name": "stdout", 328 | "output_type": "stream", 329 | "text": [ 330 | "Updating nb from WhySqrt5.swift\n", 331 | "Updating nb from Anneal.swift\n", 332 | "Updating nb from Callbacks.swift\n", 333 | "Updating nb from Optimizer.swift\n", 334 | "Updating nb from MixupLs.swift\n", 335 | "Updating nb from DataBlock.swift\n", 336 | "Updating nb from FastaiLayers.swift\n", 337 | "Updating nb from FullyConnected.swift\n", 338 | "Updating nb from HeterogeneousDictionary.swift\n", 339 | "Updating nb from Batchnorm.swift\n", 340 | "Updating nb from MinibatchTraining.swift\n", 341 | "Updating nb from LoadData.swift\n", 342 | "Updating nb from Matmul.swift\n", 343 | "Updating nb from Imagenette.swift\n", 344 | "Updating nb from EarlyStopping.swift\n", 345 | "Updating nb from Cuda.swift\n" 346 | ] 347 | } 348 | ], 349 | "source": [ 350 | "updateNotebooks(dest, nbFolder: nbFolder)" 351 | ] 352 | }, 353 | { 354 | "cell_type": "markdown", 355 | "metadata": {}, 356 | "source": [ 357 | "## Diffing" 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": null, 363 | "metadata": {}, 364 | "outputs": [], 365 | "source": [ 366 | "public func diffNbScript(_ nbFname: Path, dest: Path){\n", 367 | " let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n", 368 | " let destFname = (dest ?? fname.parent) / newName\n", 369 | " // let cells = readNb(nbFname)[\"cells\"] as! [[String:Any]]\n", 370 | " \n", 371 | " let data = try! Data(contentsOf: fname.url)\n", 372 | " let code: String = String(data: data, encoding: .utf8)!\n", 373 | " let codeCells = code.components(separatedBy: \"//cell\")\n", 374 | " let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n", 375 | " let nbData = try! Data(contentsOf: nbFname.url)\n", 376 | " var jsonData = try! JSONSerialization.jsonObject(with: nbData, options: .allowFragments) as! [String: Any]\n", 377 | " var cells = jsonData[\"cells\"] as! [[String:Any]]\n", 378 | " for c in codeCells[1...] {\n", 379 | " var lines = c.components(separatedBy: \"\\n\")\n", 380 | " let idx: Int = Int(lines[0])!\n", 381 | " var i = lines.count-1\n", 382 | " while lines[i].isEmpty { i -= 1}\n", 383 | " if i > 1 {\n", 384 | " for i in 1...(i-1) { lines[i].append(\"\\n\") }\n", 385 | " }\n", 386 | " lines[0] = \"// export\\n\"\n", 387 | " cells[idx][\"source\"] = Array(lines[...i])\n", 388 | " }\n", 389 | " jsonData[\"cells\"] = cells\n", 390 | " let outData = try! JSONSerialization.data(withJSONObject: jsonData, options: .prettyPrinted) \n", 391 | " let jsonString = String(data: outData, encoding: .utf8)! \n", 392 | " do { try jsonString.write(to: nbFname, encoding: .utf8) }\n", 393 | " catch { \"Couldn't save notebook\" }\n", 394 | "}" 395 | ] 396 | } 397 | ], 398 | "metadata": { 399 | "kernelspec": { 400 | "display_name": "Swift", 401 | "language": "swift", 402 | "name": "swift" 403 | }, 404 | "language_info": { 405 | "file_extension": ".swift", 406 | "mimetype": "text/x-swift", 407 | "name": "swift", 408 | "version": "" 409 | } 410 | }, 411 | "nbformat": 4, 412 | "nbformat_minor": 2 413 | } 414 | -------------------------------------------------------------------------------- /Sources/SwiftAI/Optimizer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | This file was autogenerated from 09_optimizer.ipynb 3 | 4 | If you edit it, be sure that: 5 | 1. there is no diff between this file and the corresponding notebook prior to editing 6 | 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook 7 | 8 | Run *** when you are done to update the notebooks with your change. 9 | */ 10 | 11 | //cell1 12 | import Path 13 | import TensorFlow 14 | 15 | //cell10 16 | public struct HyperParams { 17 | public static let lr = "learningRate" 18 | } 19 | 20 | //cell12 21 | public protocol StatDelegate { 22 | var name: String {get} 23 | var defaultHPs: [String:Float] {get} 24 | 25 | func update(_ state: inout [String:TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) 26 | } 27 | 28 | public protocol StepDelegate { 29 | var defaultHPs: [String:Float] {get} 30 | 31 | func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) 32 | } 33 | 34 | //cell14 35 | public func mergeDicts(_ dicts: inout [[String:Float]], with newDict: [String:Float]) { 36 | for i in dicts.indices { 37 | dicts[i].merge(newDict) { (_, new) in new } 38 | } 39 | } 40 | 41 | public func mergeDicts(_ dicts: inout [[String:Float]], with newDicts: [[String:Float]]) { 42 | for i in dicts.indices { 43 | dicts[i].merge(newDicts[i]) { (_, new) in new } 44 | } 45 | } 46 | 47 | //cell16 48 | extension Dictionary where Value == Int{ 49 | public init(mapFromArrays arrays: [[Key]]){ 50 | self.init(uniqueKeysWithValues: arrays.enumerated().flatMap { i, arr in arr.map { ($0, i) } }) 51 | } 52 | } 53 | 54 | extension Dictionary { 55 | public init(constant: Value, keys: [Key]){ 56 | self.init(uniqueKeysWithValues: keys.map { ($0, constant) }) 57 | } 58 | } 59 | 60 | //cell18 61 | public func initializeState(for model: Model, names: [String]) 62 | -> [WritableKeyPath: [String:TF]] { 63 | return [WritableKeyPath: [String:TF]]( 64 | constant: [String: TF](constant: TF(0), keys: names), 65 | keys: model.differentiableVectorView.keyPaths) 66 | } 67 | 68 | //cell20 69 | public class StatefulOptimizer: Optimizer { 70 | public typealias ModelKeyPath = WritableKeyPath 71 | public typealias SplitDict = [ModelKeyPath: Int] 72 | public var hpGroups: [[String:Float]] 73 | public var splitDict: SplitDict 74 | public var states: [ModelKeyPath: [String: TF]] 75 | public var stats: [StatDelegate] 76 | public var steppers: [StepDelegate] 77 | public init( 78 | for model: __shared Model, 79 | steppers: [StepDelegate], 80 | stats: [StatDelegate], 81 | hpGroups: [[String:Float]], 82 | splitArray: [[ModelKeyPath]] 83 | ) { 84 | self.hpGroups = Array(repeating: [:], count: hpGroups.count) 85 | (self.steppers,self.stats) = (steppers,stats) 86 | self.splitDict = SplitDict(mapFromArrays: splitArray) 87 | states = [:] 88 | steppers.forEach { mergeDicts(&self.hpGroups, with: $0.defaultHPs) } 89 | stats.forEach { mergeDicts(&self.hpGroups, with: $0.defaultHPs) } 90 | states = initializeState(for: model, names: stats.map { $0.name }) 91 | mergeDicts(&self.hpGroups, with: hpGroups) 92 | } 93 | 94 | public func update( 95 | _ model: inout Model, 96 | along direction: Model.TangentVector 97 | ) { 98 | var params = model.differentiableVectorView 99 | for kp in model.differentiableVectorView.keyPaths { 100 | var 𝛁p = direction[keyPath: kp] 101 | var hps = hpGroups[splitDict[kp]!] 102 | stats.forEach() { $0.update(&states[kp]!, 103 | p: params[keyPath: kp], 104 | 𝛁p: 𝛁p, 105 | hps: &hps) } 106 | steppers.forEach() { $0.update(¶ms[keyPath: kp], 107 | 𝛁p: &𝛁p, 108 | state: states[kp]!, 109 | hps: &hps) } 110 | hpGroups[splitDict[kp]!] = hps 111 | } 112 | model.move(along: params-model.differentiableVectorView) 113 | } 114 | 115 | //OPtimizer conformace can't be in a separate extension cause... Idk 116 | public var learningRate: Float { 117 | get { return hpGroups.last![HyperParams.lr]! } 118 | set { 119 | for i in hpGroups.indices {self.hpGroups[i][HyperParams.lr] = newValue } 120 | } 121 | } 122 | //For discriminative learning rates 123 | public var learningRates: [Float] { 124 | get { return hpGroups.map { $0[HyperParams.lr]! } } 125 | set { 126 | for i in hpGroups.indices {self.hpGroups[i][HyperParams.lr] = newValue[i] } 127 | } 128 | } 129 | 130 | public required init(copying other: StatefulOptimizer, to device: Device) { 131 | hpGroups = other.hpGroups 132 | splitDict = other.splitDict 133 | states = other.states //TODO, actually copy to device 134 | stats = other.stats 135 | steppers = other.steppers 136 | } 137 | } 138 | 139 | //cell22 140 | extension StatefulOptimizer{ 141 | public convenience init (for model: __shared Model, 142 | steppers: [StepDelegate], 143 | stats: [StatDelegate], 144 | hps: [String:Float]) { 145 | self.init(for: model, 146 | steppers: steppers, 147 | stats: stats, 148 | hpGroups: [hps], 149 | splitArray: [model.differentiableVectorView.keyPaths]) 150 | } 151 | } 152 | 153 | //cell24 154 | public struct SGDStep: StepDelegate { 155 | public var defaultHPs: [String: Float] { return [HyperParams.lr: 3e-3] } 156 | public init() {} 157 | public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) { 158 | p -= 𝛁p * hps[HyperParams.lr]! 159 | } 160 | } 161 | 162 | //cell30 163 | public extension HyperParams { 164 | static let wd = "weightDecay" 165 | } 166 | 167 | public struct WeightDecay: StepDelegate { 168 | public var defaultHPs: [String: Float] { return [HyperParams.wd: 0] } 169 | public init() {} 170 | public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) { 171 | p *= 1 - hps[HyperParams.lr]! * hps[HyperParams.wd]! 172 | } 173 | } 174 | 175 | //cell31 176 | public struct L2Regularization: StepDelegate { 177 | public var defaultHPs: [String: Float] { return [HyperParams.wd: 0] } 178 | public init() {} 179 | public func update(_ p: inout TF, 𝛁p: inout TF, state: [String:TF], hps: inout [String:Float]) { 180 | 𝛁p += hps[HyperParams.wd]! * p 181 | } 182 | } 183 | 184 | //cell33 185 | //Expandable enum to have tab completes/typo-proof for state variable names. 186 | public struct StateKeys { 187 | public static let avgGrad = "averageGrad" 188 | } 189 | 190 | //cell34 191 | public extension HyperParams { 192 | static let mom = "momentum" 193 | static let momDamp = "dampening" 194 | } 195 | 196 | public struct AverageGrad: StatDelegate { 197 | public var defaultHPs: [String: Float] { return [HyperParams.mom: 0.9] } 198 | public let dampened: Bool 199 | public init(dampened: Bool = false) { self.dampened = dampened } 200 | public var name: String { return StateKeys.avgGrad } 201 | public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) { 202 | state[StateKeys.avgGrad]! *= hps[HyperParams.mom]! 203 | hps[HyperParams.momDamp] = 1.0 - (dampened ? hps[HyperParams.mom]! : 0.0) 204 | state[StateKeys.avgGrad]! += hps[HyperParams.momDamp]! * 𝛁p 205 | } 206 | } 207 | 208 | //cell35 209 | public struct MomentumStep: StepDelegate { 210 | public var defaultHPs: [String: Float] = [:] 211 | public init() {} 212 | public func update(_ p: inout TF, 𝛁p: inout TF, state: [String: TF], hps: inout [String:Float]) { 213 | p -= state[StateKeys.avgGrad]! * hps[HyperParams.lr]! 214 | } 215 | } 216 | 217 | //cell43 218 | public extension HyperParams { 219 | static let ²mom = "momentumSquares" 220 | static let ²momDamp = "dampeningSquares" 221 | } 222 | 223 | public extension StateKeys { 224 | static let avgSqr = "averageSquaredGrad" 225 | } 226 | 227 | public struct AverageSquaredGrad: StatDelegate { 228 | let dampened: Bool 229 | public init(dampened: Bool = true) { self.dampened = dampened } 230 | public var name: String { return StateKeys.avgSqr } 231 | public var defaultHPs: [String: Float] { return [HyperParams.²mom: 0.99] } 232 | public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) { 233 | state[StateKeys.avgSqr]! *= hps[HyperParams.²mom]! 234 | hps[HyperParams.²momDamp] = 1.0 - (dampened ? hps[HyperParams.²mom]! : 0.0) 235 | state[StateKeys.avgSqr]! += hps[HyperParams.²momDamp]! * 𝛁p.squared() 236 | } 237 | } 238 | 239 | //cell45 240 | public extension StateKeys { 241 | static let step = "stepCount" 242 | } 243 | 244 | public struct StepCount: StatDelegate { 245 | public var name: String { return StateKeys.step } 246 | public var defaultHPs: [String:Float] = [:] 247 | public init() {} 248 | public func update(_ state: inout [String: TF], p: TF, 𝛁p: TF, hps: inout [String:Float]) { 249 | state[StateKeys.step]! += 1.0 250 | } 251 | } 252 | 253 | //cell46 254 | //public struct Epsilon: HetDictKey { public static var defaultValue: Float = 1e-5 } 255 | public extension HyperParams { 256 | static let eps = "epsilon" 257 | } 258 | 259 | //cell47 260 | public struct AdamStep: StepDelegate { 261 | public var defaultHPs: [String: Float] { return [HyperParams.eps: 1e-5] } 262 | public init() {} 263 | public func update(_ p: inout TF, 𝛁p: inout TF, state: [String: TF], hps: inout [String:Float]) { 264 | let stepCount = state[StateKeys.step]! 265 | let (mom,damp) = (hps[HyperParams.mom]!,hps[HyperParams.momDamp]!) 266 | let debias1 = damp * (1 - pow(mom, stepCount)) / (1 - mom) 267 | let num = state[StateKeys.avgGrad]!/debias1 268 | 269 | let (²mom,²damp) = (hps[HyperParams.²mom]!,hps[HyperParams.²momDamp]!) 270 | let debias2 = ²damp * (1 - pow(²mom, stepCount)) / (1 - ²mom) 271 | let denom = sqrt(state[StateKeys.avgSqr]!/debias2) + hps[HyperParams.eps]! 272 | p -= hps[HyperParams.lr]! * num / denom 273 | } 274 | } 275 | 276 | //cell58 277 | public func sgdOpt(lr: Float, mom: Float = 0.9, wd: Float = 0.0, dampening: Bool = false 278 | ) -> ((Model) -> StatefulOptimizer) { 279 | var steppers: [StepDelegate] = (mom != 0) ? [MomentumStep()] : [SGDStep()] 280 | if wd != 0 { steppers.append(WeightDecay()) } 281 | let stats = (mom != 0) ? [AverageGrad(dampened: dampening)] : [] 282 | var hps: [String: Float] = [HyperParams.lr: lr] 283 | if mom != 0 { hps[HyperParams.mom] = mom } 284 | if wd != 0 { hps[HyperParams.wd ] = wd } 285 | return {model in 286 | return StatefulOptimizer(for: model, steppers: steppers, stats: stats, hps: hps)} 287 | } 288 | 289 | //cell59 290 | public func adamOpt(lr: Float, mom: Float = 0.9, beta: Float=0.99, wd: Float = 0.0, eps: Float = 1e-5 291 | ) -> ((Model) -> StatefulOptimizer) { 292 | var steppers: [StepDelegate] = [AdamStep()] 293 | if wd != 0 { steppers.append(WeightDecay()) } 294 | let stats: [StatDelegate] = [AverageGrad(dampened: true), AverageSquaredGrad(), StepCount()] 295 | var hps: [String: Float] = [HyperParams.lr: lr] 296 | hps[HyperParams.mom] = mom 297 | hps[HyperParams.²mom] = beta 298 | hps[HyperParams.eps] = eps 299 | if wd != 0 { hps[HyperParams.wd ] = wd } 300 | return {model in 301 | return StatefulOptimizer(for: model, steppers: steppers, stats: stats, hps: hps)} 302 | } 303 | 304 | //cell62 305 | public extension StatefulOptimizer { 306 | func setParam(_ hp: String, _ val: Float) { 307 | for i in 0.. Float 316 | 317 | // A learning rate schedule from step to float. 318 | public var scheduler: ScheduleFunc 319 | public let hp: String 320 | 321 | public init(scheduler: @escaping (Float) -> Float, hp: String) { 322 | (self.scheduler,self.hp) = (scheduler,hp) 323 | } 324 | 325 | override public func batchWillStart(learner: Learner) { 326 | let val = scheduler(learner.pctEpochs/Float(learner.epochCount)) 327 | (learner.opt as! StatefulOptimizer).setParam(hp, val) 328 | } 329 | } 330 | 331 | public func makeParamScheduler(_ scheduler: @escaping (Float) -> Float, hp: String) -> ParamScheduler { 332 | return ParamScheduler(scheduler: scheduler, hp: hp) 333 | } 334 | } 335 | 336 | //cell65 337 | public func oneCycleSchedulers(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5, 338 | moms: (Float,Float,Float) = (0.95,0.85,0.95)) 339 | -> ((Float) -> Float, (Float) -> Float){ 340 | let lrSched = combineSchedules( 341 | pcts: [pctStart, 1-pctStart], 342 | schedules: [makeAnnealer(start: lrMax/divStart, end: lrMax, schedule: cosineSchedule), 343 | makeAnnealer(start: lrMax, end: lrMax/divEnd, schedule: cosineSchedule)]) 344 | let momSched = combineSchedules( 345 | pcts: [pctStart, 1-pctStart], 346 | schedules: [makeAnnealer(start: moms.0, end: moms.1, schedule: cosineSchedule), 347 | makeAnnealer(start: moms.1, end: moms.2, schedule: cosineSchedule)]) 348 | return (lrSched, momSched) 349 | } 350 | 351 | //cell66 352 | extension Learner where Opt.Scalar: BinaryFloatingPoint { 353 | 354 | public func addOneCycleDelegates(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5, 355 | moms: (Float,Float,Float) = (0.95,0.85,0.95)) { 356 | let scheds = oneCycleSchedulers(lrMax, pctStart: pctStart, divStart: divStart, divEnd: divEnd, moms: moms) 357 | addDelegates([makeParamScheduler(scheds.0 , hp: HyperParams.lr), 358 | makeParamScheduler(scheds.1 , hp: HyperParams.mom)]) 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /nbs/08a_heterogeneous_dictionary.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Installing packages:\n", 13 | "\t.package(path: \"/home/jupyter/git/fastai_dev/swift/FastaiNotebook_08_data_block\")\n", 14 | "\t\tFastaiNotebook_08_data_block\n", 15 | "With SwiftPM flags: []\n", 16 | "Working in: /tmp/tmpdolnmyrt/swift-install\n", 17 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 18 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 19 | "Updating https://github.com/mxcl/Path.swift\n", 20 | "Updating https://github.com/saeta/Just\n", 21 | "Updating https://github.com/latenitesoft/NotebookExport\n", 22 | "Completed resolution in 2.31s\n", 23 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 24 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 25 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 26 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 27 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 28 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 29 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 30 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 31 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 32 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 33 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 34 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 35 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 36 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 37 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 38 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 39 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 40 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 41 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 42 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 43 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 44 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 45 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 46 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 47 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 48 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 49 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 50 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 51 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 52 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 53 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 54 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[1/13] Compiling FastaiNotebook_08_data_block 05b_early_stopping.swift\n", 55 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 56 | "[2/13] Compiling FastaiNotebook_08_data_block 06_cuda.swift\n", 57 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 58 | "[3/13] Compiling FastaiNotebook_08_data_block 00_load_data.swift\n", 59 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 60 | "[4/13] Compiling FastaiNotebook_08_data_block 01_matmul.swift\n", 61 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 62 | "[5/13] Compiling FastaiNotebook_08_data_block 02a_why_sqrt5.swift\n", 63 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 64 | "[6/13] Compiling FastaiNotebook_08_data_block 03_minibatch_training.swift\n", 65 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 66 | "[7/13] Compiling FastaiNotebook_08_data_block 08_data_block.swift\n", 67 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 68 | "[8/13] Compiling FastaiNotebook_08_data_block 04_callbacks.swift\n", 69 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 70 | "[9/13] Compiling FastaiNotebook_08_data_block 05_anneal.swift\n", 71 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 72 | "[10/13] Compiling FastaiNotebook_08_data_block 01a_fastai_layers.swift\n", 73 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 74 | "[11/13] Compiling FastaiNotebook_08_data_block 02_fully_connected.swift\n" 75 | ] 76 | }, 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 82 | "[12/13] Compiling FastaiNotebook_08_data_block 07_batchnorm.swift\n", 83 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 84 | "[13/14] Merging module FastaiNotebook_08_data_block\n", 85 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 86 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[14/15] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", 87 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 88 | "[15/16] Merging module jupyterInstalledPackages\n", 89 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 90 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 91 | "/home/jupyter/swift/usr/bin/swift-autolink-extract: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift-autolink-extract)\n", 92 | "[16/16] Linking libjupyterInstalledPackages.so\n", 93 | "Initializing Swift...\n", 94 | "Installation complete!\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "%install-location $cwd/swift-install\n", 100 | "%install '.package(path: \"$cwd/FastaiNotebook_08_data_block\")' FastaiNotebook_08_data_block" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "//export\n", 110 | "import Path" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "import FastaiNotebook_08_data_block" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "// export\n", 129 | "public protocol HetDictKey {\n", 130 | " associatedtype ValueType\n", 131 | " static var defaultValue: ValueType { get }\n", 132 | "}" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "// export\n", 142 | "\n", 143 | "public struct HeterogeneousDictionary {\n", 144 | " private var underlying: [ObjectIdentifier : Any] = [:]\n", 145 | " \n", 146 | " public init() {}\n", 147 | " public init(_ key: T.Type, _ value: T.ValueType) {\n", 148 | " self.underlying = [ObjectIdentifier(key): value]\n", 149 | " }\n", 150 | " public init(_ key1: T1.Type, _ value1: T1.ValueType, _ key2: T2.Type, _ value2: T2.ValueType) {\n", 151 | " self.underlying = [ObjectIdentifier(key1): value1, ObjectIdentifier(key2): value2]\n", 152 | " }\n", 153 | "\n", 154 | " public subscript(key: T.Type) -> T.ValueType {\n", 155 | " get { return underlying[ObjectIdentifier(key), default: T.defaultValue] as! T.ValueType }\n", 156 | " set { underlying[ObjectIdentifier(key)] = newValue as Any }\n", 157 | " }\n", 158 | " \n", 159 | " public mutating func merge(_ other: HeterogeneousDictionary,\n", 160 | " uniquingKeysWith combine: (Any, Any) throws -> Any) rethrows {\n", 161 | " try self.underlying.merge(other.underlying, uniquingKeysWith: combine)\n", 162 | " }\n", 163 | "}\n" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "// export\n", 173 | "// Common keys\n", 174 | "public struct LearningRate: HetDictKey {\n", 175 | " public static var defaultValue: Float = 0.4\n", 176 | "}" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "public struct StepCount: HetDictKey {\n", 186 | " public static var defaultValue = 0\n", 187 | "}" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "// Sample usage\n", 197 | "var m = HeterogeneousDictionary()\n" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": {}, 204 | "outputs": [ 205 | { 206 | "name": "stdout", 207 | "output_type": "stream", 208 | "text": [ 209 | "0.4\r\n", 210 | "3.4\r\n", 211 | "0\r\n", 212 | "3\r\n" 213 | ] 214 | } 215 | ], 216 | "source": [ 217 | "print(m[LearningRate.self])\n", 218 | "m[LearningRate.self] = 3.4\n", 219 | "print(m[LearningRate.self])\n", 220 | "\n", 221 | "print(m[StepCount.self])\n", 222 | "m[StepCount.self] = 3\n", 223 | "print(m[StepCount.self])\n" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "name": "stdout", 233 | "output_type": "stream", 234 | "text": [ 235 | "Int\r\n", 236 | "Float\r\n" 237 | ] 238 | } 239 | ], 240 | "source": [ 241 | "print(type(of: m[StepCount.self]))\n", 242 | "print(type(of: m[LearningRate.self]))\n" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "## Export" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "name": "stdout", 259 | "output_type": "stream", 260 | "text": [ 261 | "success\r\n" 262 | ] 263 | } 264 | ], 265 | "source": [ 266 | "import NotebookExport\n", 267 | "let exporter = NotebookExport(Path.cwd/\"08a_heterogeneous_dictionary.ipynb\")\n", 268 | "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [] 277 | } 278 | ], 279 | "metadata": { 280 | "kernelspec": { 281 | "display_name": "Swift", 282 | "language": "swift", 283 | "name": "swift" 284 | } 285 | }, 286 | "nbformat": 4, 287 | "nbformat_minor": 2 288 | } 289 | -------------------------------------------------------------------------------- /tools/.ipynb_checkpoints/export_import-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Installing packages:\n", 13 | "\t.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")\n", 14 | "\t\tPath\n", 15 | "With SwiftPM flags: []\n", 16 | "Working in: /tmp/tmp8x665_rc/swift-install\n", 17 | "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", 18 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 19 | "Fetching https://github.com/mxcl/Path.swift\n", 20 | "Completed resolution in 0.65s\n", 21 | "Cloning https://github.com/mxcl/Path.swift\n", 22 | "Resolving https://github.com/mxcl/Path.swift at 0.16.3\n", 23 | "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", 24 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 25 | "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", 26 | "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[1/10] Compiling Path Path+StringConvertibles.swift\n", 27 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 28 | "[2/10] Compiling Path Extensions.swift\n", 29 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 30 | "[3/10] Compiling Path Path+Attributes.swift\n", 31 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 32 | "[4/10] Compiling Path Path->Bool.swift\n", 33 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 34 | "[5/10] Compiling Path Path+Codable.swift\n", 35 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 36 | "[6/10] Compiling Path Path+ls.swift\n", 37 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 38 | "[7/10] Compiling Path Path+CommonDirectories.swift\n", 39 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 40 | "[8/10] Compiling Path Path+FileManager.swift\n", 41 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 42 | "[9/10] Compiling Path Path.swift\n", 43 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 44 | "[10/11] Merging module Path\n", 45 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 46 | "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[11/12] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", 47 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 48 | "[12/13] Merging module jupyterInstalledPackages\n", 49 | "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n", 50 | "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n", 51 | "/home/sgugger/swift/usr/bin/swift-autolink-extract: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift-autolink-extract)\n", 52 | "[13/13] Linking libjupyterInstalledPackages.so\n", 53 | "Initializing Swift...\n", 54 | "Installation complete!\n" 55 | ] 56 | } 57 | ], 58 | "source": [ 59 | "%install-location $cwd/swift-install\n", 60 | "%install '.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")' Path" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 2, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "import Foundation\n", 70 | "import Path" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "/home/sgugger/git/swiftai\r\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "let path = Path.cwd.parent\n", 88 | "print(path)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 4, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "public extension String {\n", 98 | " func findFirst(pat: String) -> Range? {\n", 99 | " return range(of: pat, options: .regularExpression)\n", 100 | " }\n", 101 | " func hasMatch(pat: String) -> Bool {\n", 102 | " return findFirst(pat:pat) != nil\n", 103 | " }\n", 104 | " func withMaj() -> String {\n", 105 | " return prefix(1).capitalized + dropFirst()\n", 106 | " }\n", 107 | "}" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "## Notebook to script" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 5, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "public func nbNameToScriptName(_ name: String) -> String {\n", 124 | " var splits = name.components(separatedBy: \"_\")\n", 125 | " splits = splits[1...].map { $0.withMaj() }\n", 126 | " return splits.joined(separator: \"\")\n", 127 | "}" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 6, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "public func readNb(_ fname: Path) -> [String:Any] {\n", 137 | " let data = try! Data(contentsOf: fname.url)\n", 138 | " return try! JSONSerialization.jsonObject(with: data, options: .allowFragments) as! [String: Any]\n", 139 | "}" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 7, 145 | "metadata": {}, 146 | "outputs": [], 147 | "source": [ 148 | "public func notebookToScript(_ fname: Path, dest: Path?=nil) {\n", 149 | " let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n", 150 | " let destFname = (dest ?? fname.parent) / newName\n", 151 | " let cells = readNb(fname)[\"cells\"] as! [[String:Any]]\n", 152 | " var module = \"\"\"\n", 153 | "/*\n", 154 | "This file was autogenerated from \\(fname.basename())\n", 155 | " \n", 156 | "If you edit it, be sure that:\n", 157 | " 1. there is no diff between this file and the corresponding notebook prior to editing\n", 158 | " 2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook\n", 159 | " \n", 160 | "Run *** when you are done to update the notebooks with your change.\n", 161 | "*/\n", 162 | " \n", 163 | "\"\"\"\n", 164 | " for (i,cell) in cells.enumerated() {\n", 165 | " if let source = cell[\"source\"] as? [String], !source.isEmpty, source[0].hasMatch(pat: #\"^\\s*//\\s*export\\s*$\"#) {\n", 166 | " module.append(\"\\n//cell\\(i)\\n\\(source[1...].joined())\\n\")\n", 167 | " }\n", 168 | " }\n", 169 | " try! module.write(to: destFname, encoding: .utf8)\n", 170 | "}" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 9, 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "let fname = path/\"/nbs/00_load_data.ipynb\"\n", 180 | "let dest = path/\"Sources/SwiftAI\"" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 10, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "notebookToScript(fname, dest: dest)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 11, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "public func makeLibrary(_ nbFolder: Path, dest: Path?=nil){\n", 199 | " for entry in try! nbFolder.ls() where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\"^\\d+[a-z]*_.*ipynb$\"#) {\n", 200 | " print(\"Converting \\(entry.path.basename())\")\n", 201 | " notebookToScript(entry.path, dest: dest)\n", 202 | " }\n", 203 | "}" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 12, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "let nbFolder = path/\"nbs\"" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 13, 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "name": "stdout", 222 | "output_type": "stream", 223 | "text": [ 224 | "Converting 02a_why_sqrt5.ipynb\n", 225 | "Converting 10_mixup_ls.ipynb\n", 226 | "Converting 05_anneal.ipynb\n", 227 | "Converting 09_optimizer.ipynb\n", 228 | "Converting 04_callbacks.ipynb\n", 229 | "Converting 08_data_block.ipynb\n", 230 | "Converting 08a_heterogeneous_dictionary.ipynb\n", 231 | "Converting 06_cuda.ipynb\n", 232 | "Converting 01_matmul.ipynb\n", 233 | "Converting 02_fully_connected.ipynb\n", 234 | "Converting 01a_fastai_layers.ipynb\n", 235 | "Converting 11_imagenette.ipynb\n", 236 | "Converting 05b_early_stopping.ipynb\n", 237 | "Converting 00_load_data.ipynb\n", 238 | "Converting 03_minibatch_training.ipynb\n", 239 | "Converting 07_batchnorm.ipynb\n" 240 | ] 241 | } 242 | ], 243 | "source": [ 244 | "makeLibrary(nbFolder, dest:dest)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "## Script to notebook" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": 13, 257 | "metadata": {}, 258 | "outputs": [], 259 | "source": [ 260 | "let fname = dest/\"LoadData.swift\"" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": 14, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "public func readScript(_ fname: Path) -> String {\n", 270 | " let data = try! Data(contentsOf: fname.url)\n", 271 | " return String(data: data, encoding: .utf8)!\n", 272 | "}" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 15, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "public func writeNotebook(_ nbFname: Path, nbData: [String: Any]) {\n", 282 | " let outData = try! JSONSerialization.data(withJSONObject: nbData, options: .prettyPrinted) \n", 283 | " let jsonString = String(data: outData, encoding: .utf8)! \n", 284 | " do { try jsonString.write(to: nbFname, encoding: .utf8) }\n", 285 | " catch { \"Couldn't save notebook\" }\n", 286 | "}" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 18, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "public func scriptToNotebook(_ fname: Path, nbFolder: Path){\n", 296 | " let code = readScript(fname)\n", 297 | " let codeCells = code.components(separatedBy: \"//cell\")\n", 298 | " let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n", 299 | " let nbFname = nbFolder / nbName\n", 300 | " var jsonData = readNb(nbFname)\n", 301 | " var cells = jsonData[\"cells\"] as! [[String:Any]]\n", 302 | " for c in codeCells[1...] {\n", 303 | " var lines = c.components(separatedBy: \"\\n\")\n", 304 | " let idx: Int = Int(lines[0])!\n", 305 | " var i = lines.count-1\n", 306 | " while lines[i].isEmpty { i -= 1}\n", 307 | " if i > 1 {\n", 308 | " for i in 1...(i-1) { lines[i].append(\"\\n\") }\n", 309 | " }\n", 310 | " lines[0] = \"// export\\n\"\n", 311 | " cells[idx][\"source\"] = Array(lines[...i])\n", 312 | " }\n", 313 | " jsonData[\"cells\"] = cells\n", 314 | " writeNotebook(nbFname, nbData: jsonData)\n", 315 | "}" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 19, 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "scriptToNotebook(fname, nbFolder: nbFolder)" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": 20, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "public func updateNotebooks(_ scriptsFolder: Path, nbFolder: Path) {\n", 334 | " for entry in try! scriptsFolder.ls() where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\".swift$\"#) {\n", 335 | " print(\"Updating nb from \\(entry.path.basename())\")\n", 336 | " scriptToNotebook(entry.path, nbFolder: nbFolder)\n", 337 | " }\n", 338 | "}" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 21, 344 | "metadata": {}, 345 | "outputs": [ 346 | { 347 | "name": "stdout", 348 | "output_type": "stream", 349 | "text": [ 350 | "Updating nb from WhySqrt5.swift\n", 351 | "Updating nb from Anneal.swift\n", 352 | "Updating nb from Callbacks.swift\n", 353 | "Updating nb from Optimizer.swift\n", 354 | "Updating nb from MixupLs.swift\n", 355 | "Updating nb from DataBlock.swift\n", 356 | "Updating nb from FastaiLayers.swift\n", 357 | "Updating nb from FullyConnected.swift\n", 358 | "Updating nb from HeterogeneousDictionary.swift\n", 359 | "Updating nb from Batchnorm.swift\n", 360 | "Updating nb from MinibatchTraining.swift\n", 361 | "Updating nb from LoadData.swift\n", 362 | "Updating nb from Matmul.swift\n", 363 | "Updating nb from Imagenette.swift\n", 364 | "Updating nb from EarlyStopping.swift\n", 365 | "Updating nb from Cuda.swift\n" 366 | ] 367 | } 368 | ], 369 | "source": [ 370 | "updateNotebooks(dest, nbFolder: nbFolder)" 371 | ] 372 | }, 373 | { 374 | "cell_type": "markdown", 375 | "metadata": {}, 376 | "source": [ 377 | "## Diffing" 378 | ] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [ 386 | "public func diffNbScript(_ nbFname: Path, dest: Path){\n", 387 | " let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n", 388 | " let destFname = (dest ?? fname.parent) / newName\n", 389 | " let cells = readNb(nbFname)[\"cells\"] as! [[String:Any]]\n", 390 | " \n", 391 | " let data = try! Data(contentsOf: fname.url)\n", 392 | " let code: String = String(data: data, encoding: .utf8)!\n", 393 | " let codeCells = code.components(separatedBy: \"//cell\")\n", 394 | " let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n", 395 | " let nbData = try! Data(contentsOf: nbFname.url)\n", 396 | " var jsonData = try! JSONSerialization.jsonObject(with: nbData, options: .allowFragments) as! [String: Any]\n", 397 | " var cells = jsonData[\"cells\"] as! [[String:Any]]\n", 398 | " for c in codeCells[1...] {\n", 399 | " var lines = c.components(separatedBy: \"\\n\")\n", 400 | " let idx: Int = Int(lines[0])!\n", 401 | " var i = lines.count-1\n", 402 | " while lines[i].isEmpty { i -= 1}\n", 403 | " if i > 1 {\n", 404 | " for i in 1...(i-1) { lines[i].append(\"\\n\") }\n", 405 | " }\n", 406 | " lines[0] = \"// export\\n\"\n", 407 | " cells[idx][\"source\"] = Array(lines[...i])\n", 408 | " }\n", 409 | " jsonData[\"cells\"] = cells\n", 410 | " let outData = try! JSONSerialization.data(withJSONObject: jsonData, options: .prettyPrinted) \n", 411 | " let jsonString = String(data: outData, encoding: .utf8)! \n", 412 | " do { try jsonString.write(to: nbFname, encoding: .utf8) }\n", 413 | " catch { \"Couldn't save notebook\" }\n", 414 | "}" 415 | ] 416 | } 417 | ], 418 | "metadata": { 419 | "kernelspec": { 420 | "display_name": "Swift", 421 | "language": "swift", 422 | "name": "swift" 423 | }, 424 | "language_info": { 425 | "file_extension": ".swift", 426 | "mimetype": "text/x-swift", 427 | "name": "swift", 428 | "version": "" 429 | } 430 | }, 431 | "nbformat": 4, 432 | "nbformat_minor": 2 433 | } 434 | -------------------------------------------------------------------------------- /nbs/02a_why_sqrt5.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Installing packages:\n", 13 | "\t.package(path: \"/home/jupyter/git/fastai_dev/swift/FastaiNotebook_02_fully_connected\")\n", 14 | "\t\tFastaiNotebook_02_fully_connected\n", 15 | "With SwiftPM flags: []\n", 16 | "Working in: /tmp/tmpzfv5xzmk/swift-install\n", 17 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 18 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 19 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 20 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 21 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 22 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 23 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 24 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 25 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 26 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 27 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 28 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 29 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 30 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 31 | "warning: /home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 32 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 33 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 34 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[1/4] Compiling FastaiNotebook_01a_fastai_layers 01_matmul.swift\n", 35 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 36 | "[2/4] Compiling FastaiNotebook_01a_fastai_layers 00_load_data.swift\n", 37 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 38 | "[3/4] Compiling FastaiNotebook_01a_fastai_layers 01a_fastai_layers.swift\n", 39 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 40 | "[4/5] Merging module FastaiNotebook_01a_fastai_layers\n", 41 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 42 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[5/9] Compiling FastaiNotebook_02_fully_connected 01_matmul.swift\n", 43 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 44 | "[6/9] Compiling FastaiNotebook_02_fully_connected 02_fully_connected.swift\n", 45 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 46 | "[7/9] Compiling FastaiNotebook_02_fully_connected 00_load_data.swift\n", 47 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 48 | "[8/9] Compiling FastaiNotebook_02_fully_connected 01a_fastai_layers.swift\n", 49 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 50 | "[9/10] Merging module FastaiNotebook_02_fully_connected\n", 51 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 52 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)[10/11] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n", 53 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 54 | "[11/12] Merging module jupyterInstalledPackages\n", 55 | "/home/jupyter/swift/usr/bin/swift: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift)\n", 56 | "/home/jupyter/swift/usr/bin/swiftc: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swiftc)\n", 57 | "/home/jupyter/swift/usr/bin/swift-autolink-extract: /home/jupyter/anaconda3/lib/libuuid.so.1: no version information available (required by /home/jupyter/swift/usr/bin/swift-autolink-extract)\n", 58 | "[12/12] Linking libjupyterInstalledPackages.so\n", 59 | "Initializing Swift...\n", 60 | "Installation complete!\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "%install-location $cwd/swift-install\n", 66 | "%install '.package(path: \"$cwd/FastaiNotebook_02_fully_connected\")' FastaiNotebook_02_fully_connected" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "//export\n", 76 | "import Foundation\n", 77 | "import TensorFlow\n", 78 | "import Path" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "import FastaiNotebook_02_fully_connected" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "## Does nn.Conv2d init work well?" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "var (xTrain, yTrain, xValid, yValid) = loadMNIST(path: Path.home/\".fastai\"/\"data\"/\"mnist_tst\")\n", 104 | "let (trainMean, trainStd) = (xTrain.mean(), xTrain.standardDeviation())\n", 105 | "xTrain = normalize(xTrain, mean: trainMean, std: trainStd)\n", 106 | "xValid = normalize(xValid, mean: trainMean, std: trainStd)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "[60000, 28, 28, 1] [10000, 28, 28, 1]\r\n" 119 | ] 120 | } 121 | ], 122 | "source": [ 123 | "xTrain = xTrain.reshaped(to: [xTrain.shape[0], 28, 28, 1])\n", 124 | "xValid = xValid.reshaped(to: [xValid.shape[0], 28, 28, 1])\n", 125 | "print(xTrain.shape, xValid.shape)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "let images = xTrain.shape[0]\n", 135 | "let classes = xValid.max() + 1\n", 136 | "let channels = 32" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "var layer1 = FAConv2D(filterShape: (5, 5, 1, channels)) //Conv2D(1, nh, 5)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "let x = xValid[0..<100]" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [ 162 | { 163 | "data": { 164 | "text/plain": [ 165 | "▿ [100, 28, 28, 1]\n", 166 | " ▿ dimensions : 4 elements\n", 167 | " - 0 : 100\n", 168 | " - 1 : 28\n", 169 | " - 2 : 28\n", 170 | " - 3 : 1\n" 171 | ] 172 | }, 173 | "execution_count": null, 174 | "metadata": {}, 175 | "output_type": "execute_result" 176 | } 177 | ], 178 | "source": [ 179 | "x.shape" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "extension Tensor where Scalar: TensorFlowFloatingPoint {\n", 189 | " func stats() -> (mean: Tensor, std: Tensor) {\n", 190 | " return (mean: mean(), std: standardDeviation())\n", 191 | " }\n", 192 | "}" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "data": { 202 | "text/plain": [ 203 | "▿ 2 elements\n", 204 | " ▿ filter : 2 elements\n", 205 | " - mean : -0.0048739817\n", 206 | " - std : 0.20316689\n", 207 | " ▿ bias : 2 elements\n", 208 | " - mean : 0.0\n", 209 | " - std : 0.0\n" 210 | ] 211 | }, 212 | "execution_count": null, 213 | "metadata": {}, 214 | "output_type": "execute_result" 215 | } 216 | ], 217 | "source": [ 218 | "(filter: layer1.filter.stats(), bias: layer1.bias.stats())" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "withDevice(.cpu){\n", 228 | " let result = layer1(x)\n", 229 | "}" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "let result = layer1(x)" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [ 246 | { 247 | "data": { 248 | "text/plain": [ 249 | "▿ 2 elements\n", 250 | " - mean : 0.0013046617\n", 251 | " - std : 0.958224\n" 252 | ] 253 | }, 254 | "execution_count": null, 255 | "metadata": {}, 256 | "output_type": "execute_result" 257 | } 258 | ], 259 | "source": [ 260 | "result.stats()" 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": {}, 266 | "source": [ 267 | "This is in 1a now so this code is disabled from here:\n", 268 | "\n", 269 | "```swift\n", 270 | "var rng = PhiloxRandomNumberGenerator.global\n", 271 | "\n", 272 | "extension Tensor where Scalar: TensorFlowFloatingPoint {\n", 273 | " init(kaimingNormal shape: TensorShape, negativeSlope: Double = 1.0) {\n", 274 | " // Assumes Leaky ReLU nonlinearity\n", 275 | " let gain = Scalar(sqrt(2.0 / (1.0 + pow(negativeSlope, 2))))\n", 276 | " let spatialDimCount = shape.count - 2\n", 277 | " let receptiveField = shape[0..(\n", 319 | " _ x: Tensor,\n", 320 | " negativeSlope: Double = 0.0\n", 321 | ") -> Tensor {\n", 322 | " return max(0, x) + T(negativeSlope) * min(0, x)\n", 323 | "}" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": null, 329 | "metadata": {}, 330 | "outputs": [ 331 | { 332 | "data": { 333 | "text/plain": [ 334 | "▿ 2 elements\n", 335 | " - mean : 0.41605353\n", 336 | " - std : 0.81414527\n" 337 | ] 338 | }, 339 | "execution_count": null, 340 | "metadata": {}, 341 | "output_type": "execute_result" 342 | } 343 | ], 344 | "source": [ 345 | "layer1.filter = Tensor(kaimingNormal: layer1.filter.shape, negativeSlope: 0.0)\n", 346 | "leakyRelu(layer1(x)).stats()" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": {}, 353 | "outputs": [ 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "▿ 2 elements\n", 358 | " - mean : 0.30673978\n", 359 | " - std : 0.5740978\n" 360 | ] 361 | }, 362 | "execution_count": null, 363 | "metadata": {}, 364 | "output_type": "execute_result" 365 | } 366 | ], 367 | "source": [ 368 | "var layer1 = FAConv2D(filterShape: (5, 5, 1, channels)) //Conv2D(1, nh, 5)\n", 369 | "leakyRelu(layer1(x)).stats()" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": null, 375 | "metadata": {}, 376 | "outputs": [ 377 | { 378 | "data": { 379 | "text/plain": [ 380 | "▿ [5, 5, 1, 32]\n", 381 | " ▿ dimensions : 4 elements\n", 382 | " - 0 : 5\n", 383 | " - 1 : 5\n", 384 | " - 2 : 1\n", 385 | " - 3 : 32\n" 386 | ] 387 | }, 388 | "execution_count": null, 389 | "metadata": {}, 390 | "output_type": "execute_result" 391 | } 392 | ], 393 | "source": [ 394 | "layer1.filter.shape" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": null, 400 | "metadata": {}, 401 | "outputs": [ 402 | { 403 | "data": { 404 | "text/plain": [ 405 | "25\n" 406 | ] 407 | }, 408 | "execution_count": null, 409 | "metadata": {}, 410 | "output_type": "execute_result" 411 | } 412 | ], 413 | "source": [ 414 | "let spatialDimCount = layer1.filter.rank - 2\n", 415 | "let receptiveField = layer1.filter.shape[0.. Double {\n", 464 | " return sqrt(2.0 / (1.0 + pow(negativeSlope, 2.0)))\n", 465 | "}" 466 | ] 467 | }, 468 | { 469 | "cell_type": "code", 470 | "execution_count": null, 471 | "metadata": {}, 472 | "outputs": [ 473 | { 474 | "data": { 475 | "text/plain": [ 476 | "▿ 5 elements\n", 477 | " - .0 : 1.0\n", 478 | " - .1 : 1.4142135623730951\n", 479 | " - .2 : 1.4141428569978354\n", 480 | " - .3 : 1.4071950894605838\n", 481 | " - .4 : 0.5773502691896257\n" 482 | ] 483 | }, 484 | "execution_count": null, 485 | "metadata": {}, 486 | "output_type": "execute_result" 487 | } 488 | ], 489 | "source": [ 490 | "(gain(1.0), gain(0.0), gain(0.01), gain(0.1), gain(sqrt(5.0)))" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": null, 496 | "metadata": {}, 497 | "outputs": [ 498 | { 499 | "data": { 500 | "text/plain": [ 501 | "0.577814\n" 502 | ] 503 | }, 504 | "execution_count": null, 505 | "metadata": {}, 506 | "output_type": "execute_result" 507 | } 508 | ], 509 | "source": [ 510 | "(2 * Tensor(randomUniform: [10000]) - 1).standardDeviation()" 511 | ] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "metadata": {}, 517 | "outputs": [ 518 | { 519 | "data": { 520 | "text/plain": [ 521 | "0.5773502691896258\n" 522 | ] 523 | }, 524 | "execution_count": null, 525 | "metadata": {}, 526 | "output_type": "execute_result" 527 | } 528 | ], 529 | "source": [ 530 | "1.0 / sqrt(3.0)" 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": null, 536 | "metadata": {}, 537 | "outputs": [], 538 | "source": [ 539 | "//export\n", 540 | "extension Tensor where Scalar: TensorFlowFloatingPoint {\n", 541 | " init(kaimingUniform shape: TensorShape, negativeSlope: Double = 1.0) {\n", 542 | " // Assumes Leaky ReLU nonlinearity\n", 543 | " let gain = Scalar.init(TensorFlow.sqrt(2.0 / (1.0 + TensorFlow.pow(negativeSlope, 2))))\n", 544 | " let spatialDimCount = shape.count - 2\n", 545 | " let receptiveField = shape[0..(\n", 607 | " filterShape: (5, 5, 1, 8), strides: (2, 2), padding: .same, activation: relu\n", 608 | " )\n", 609 | " public var conv2 = FAConv2D(\n", 610 | " filterShape: (3, 3, 8, 16), strides: (2, 2), padding: .same, activation: relu\n", 611 | " )\n", 612 | " public var conv3 = FAConv2D(\n", 613 | " filterShape: (3, 3, 16, 32), strides: (2, 2), padding: .same, activation: relu\n", 614 | " )\n", 615 | " public var conv4 = FAConv2D(\n", 616 | " filterShape: (3, 3, 32, 1), strides: (2, 2), padding: .valid\n", 617 | " )\n", 618 | " public var flatten = Flatten()\n", 619 | "\n", 620 | " @differentiable\n", 621 | " public func callAsFunction(_ input: Tensor) -> Tensor {\n", 622 | " return input.sequenced(through: conv1, conv2, conv3, conv4, flatten)\n", 623 | " }\n", 624 | "}" 625 | ] 626 | }, 627 | { 628 | "cell_type": "code", 629 | "execution_count": null, 630 | "metadata": {}, 631 | "outputs": [], 632 | "source": [ 633 | "let y = Tensor(yValid[0..<100])\n", 634 | "var model = Model()" 635 | ] 636 | }, 637 | { 638 | "cell_type": "code", 639 | "execution_count": null, 640 | "metadata": {}, 641 | "outputs": [ 642 | { 643 | "data": { 644 | "text/plain": [ 645 | "▿ 2 elements\n", 646 | " - mean : 0.18727446\n", 647 | " - std : 0.15831667\n" 648 | ] 649 | }, 650 | "execution_count": null, 651 | "metadata": {}, 652 | "output_type": "execute_result" 653 | } 654 | ], 655 | "source": [ 656 | "let prediction = model(x)\n", 657 | "prediction.stats()" 658 | ] 659 | }, 660 | { 661 | "cell_type": "code", 662 | "execution_count": null, 663 | "metadata": {}, 664 | "outputs": [ 665 | { 666 | "data": { 667 | "text/plain": [ 668 | "▿ 2 elements\n", 669 | " - mean : -0.052186128\n", 670 | " - std : 0.33932656\n" 671 | ] 672 | }, 673 | "execution_count": null, 674 | "metadata": {}, 675 | "output_type": "execute_result" 676 | } 677 | ], 678 | "source": [ 679 | "let gradients = gradient(at: model) { model in\n", 680 | " meanSquaredError(predicted: model(x), expected: y)\n", 681 | "}\n", 682 | "\n", 683 | "gradients.conv1.filter.stats()" 684 | ] 685 | }, 686 | { 687 | "cell_type": "code", 688 | "execution_count": null, 689 | "metadata": {}, 690 | "outputs": [], 691 | "source": [ 692 | "for keyPath in [\\Model.conv1, \\Model.conv2, \\Model.conv3, \\Model.conv4] {\n", 693 | " model[keyPath: keyPath].filter = Tensor(kaimingUniform: model[keyPath: keyPath].filter.shape)\n", 694 | "}" 695 | ] 696 | }, 697 | { 698 | "cell_type": "code", 699 | "execution_count": null, 700 | "metadata": {}, 701 | "outputs": [ 702 | { 703 | "data": { 704 | "text/plain": [ 705 | "▿ 2 elements\n", 706 | " - mean : -0.07902523\n", 707 | " - std : 0.20309263\n" 708 | ] 709 | }, 710 | "execution_count": null, 711 | "metadata": {}, 712 | "output_type": "execute_result" 713 | } 714 | ], 715 | "source": [ 716 | "let prediction = model(x)\n", 717 | "prediction.stats()" 718 | ] 719 | }, 720 | { 721 | "cell_type": "code", 722 | "execution_count": null, 723 | "metadata": {}, 724 | "outputs": [ 725 | { 726 | "data": { 727 | "text/plain": [ 728 | "▿ 2 elements\n", 729 | " - mean : -0.033189915\n", 730 | " - std : 0.3524679\n" 731 | ] 732 | }, 733 | "execution_count": null, 734 | "metadata": {}, 735 | "output_type": "execute_result" 736 | } 737 | ], 738 | "source": [ 739 | "let gradients = gradient(at: model) { model in\n", 740 | " meanSquaredError(predicted: model(x), expected: y)\n", 741 | "}\n", 742 | "\n", 743 | "gradients.conv1.filter.stats()" 744 | ] 745 | }, 746 | { 747 | "cell_type": "markdown", 748 | "metadata": {}, 749 | "source": [ 750 | "## Export" 751 | ] 752 | }, 753 | { 754 | "cell_type": "code", 755 | "execution_count": null, 756 | "metadata": {}, 757 | "outputs": [ 758 | { 759 | "name": "stdout", 760 | "output_type": "stream", 761 | "text": [ 762 | "success\r\n" 763 | ] 764 | } 765 | ], 766 | "source": [ 767 | "import NotebookExport\n", 768 | "let exporter = NotebookExport(Path.cwd/\"02a_why_sqrt5.ipynb\")\n", 769 | "print(exporter.export(usingPrefix: \"FastaiNotebook_\"))" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": null, 775 | "metadata": {}, 776 | "outputs": [], 777 | "source": [] 778 | } 779 | ], 780 | "metadata": { 781 | "kernelspec": { 782 | "display_name": "Swift", 783 | "language": "swift", 784 | "name": "swift" 785 | } 786 | }, 787 | "nbformat": 4, 788 | "nbformat_minor": 1 789 | } 790 | --------------------------------------------------------------------------------