├── .gitignore ├── CTR_NN ├── README.md ├── ctr_data.txt ├── deep nn ctr prediction │ ├── CTR_prediction_LR_FM_CCPM_PNN.ipynb │ ├── DeepFM.py │ ├── DeepFM_NFM_DeepCTR.ipynb │ ├── run_dfm.ipynb │ ├── wide_and_deep_model_criteo.ipynb │ └── wide_and_deep_model_official.ipynb └── 从FM推演各深度学习CTR预估模型（附代码）.pdf ├── DeepRL-Agents ├── A3C-Doom.ipynb ├── Contextual-Policy.ipynb ├── Deep-Recurrent-Q-Network.ipynb ├── Double-Dueling-DQN.ipynb ├── LICENSE ├── Model-Network.ipynb ├── Policy-Network.ipynb ├── Q-Exploration.ipynb ├── Q-Network.ipynb ├── Q-Table.ipynb ├── README.md ├── Simple-Policy.ipynb ├── Vanilla-Policy.ipynb ├── basic.wad ├── gridworld.py └── helper.py ├── Hvass-Labs_TensorFlow-Tutorials ├── 01_Simple_Linear_Model.ipynb ├── 02_Convolutional_Neural_Network.ipynb ├── 03B_Layers_API.ipynb ├── 03C_Keras_API.ipynb ├── 03_PrettyTensor.ipynb ├── 04_Save_Restore.ipynb ├── 05_Ensemble_Learning.ipynb ├── 06_CIFAR-10.ipynb ├── 07_Inception_Model.ipynb ├── 08_Transfer_Learning.ipynb ├── 09_Video_Data.ipynb ├── 10_Fine-Tuning.ipynb ├── 11_Adversarial_Examples.ipynb ├── 12_Adversarial_Noise_MNIST.ipynb ├── 13B_Visual_Analysis_MNIST.ipynb ├── 13_Visual_Analysis.ipynb ├── 14_DeepDream.ipynb ├── 15_Style_Transfer.ipynb ├── 16_Reinforcement_Learning.ipynb ├── 17_Estimator_API.ipynb ├── 18_TFRecords_Dataset_API.ipynb ├── cache.py ├── cifar10.py ├── convert.py ├── dataset.py ├── download.py ├── forks.md ├── inception.py ├── inception5h.py ├── knifey.py ├── reinforcement_learning.py ├── requirements.txt └── vgg16.py ├── Learning-PySpark ├── .gitattributes ├── .gitignore ├── BonusChapter01 │ └── HelloWorldFromPySpark.ipynb ├── Chapter01 │ └── LearningPySpark_Chapter02.ipynb ├── Chapter02 │ └── LearningPySpark_Chapter03.ipynb ├── Chapter03 │ └── LearningPySpark_Chapter04.ipynb ├── Chapter04 │ └── LearningPySpark_Chapter05.ipynb ├── Chapter05 │ └── LearningPySpark_Chapter06.ipynb ├── Chapter06 │ └── LearningPySpark_Chapter07.ipynb ├── Chapter07 │ └── LearningPySpark_Chapter08.ipynb ├── Chapter08 │ └── LearningPySpark_Chapter09.ipynb ├── Chapter09 │ ├── stateful_streaming_word_count.py │ ├── streaming_word_count.py │ └── structured_streaming_word_count.py ├── Chapter10 │ ├── additionalCode │ │ ├── setup.py │ │ └── utilities │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── converters │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ └── distance.py │ │ │ └── geoCalc.py │ ├── calculatingGeoDistance.py │ └── launch_spark_submit.sh ├── LICENSE └── README.md ├── PairCNN-Ranking ├── Ranking.py ├── data │ ├── test.txt │ └── train.txt └── train.py ├── README.md ├── Reinforcement-learning-with-tensorflow-master ├── README.md ├── RL_cover.jpg ├── contents │ ├── 10_A3C │ │ ├── A3C_RNN.py │ │ ├── A3C_continuous_action.py │ │ └── A3C_discrete_action.py │ ├── 11_Dyna_Q │ │ ├── RL_brain.py │ │ ├── maze_env.py │ │ └── run_this.py │ ├── 12_Proximal_Policy_Optimization │ │ ├── DPPO.py │ │ └── simply_PPO.py │ ├── 1_command_line_reinforcement_learning │ │ └── treasure_on_right.py │ ├── 2_Q_Learning_maze │ │ ├── RL_brain.py │ │ ├── maze_env.py │ │ └── run_this.py │ ├── 3_Sarsa_maze │ │ ├── RL_brain.py │ │ ├── maze_env.py │ │ └── run_this.py │ ├── 4_Sarsa_lambda_maze │ │ ├── RL_brain.py │ │ ├── maze_env.py │ │ └── run_this.py │ ├── 5.1_Double_DQN │ │ ├── RL_brain.py │ │ └── run_Pendulum.py │ ├── 5.2_Prioritized_Replay_DQN │ │ ├── RL_brain.py │ │ └── run_MountainCar.py │ ├── 5.3_Dueling_DQN │ │ ├── RL_brain.py │ │ └── run_Pendulum.py │ ├── 5_Deep_Q_Network │ │ ├── DQN_modified.py │ │ ├── RL_brain.py │ │ ├── maze_env.py │ │ └── run_this.py │ ├── 6_OpenAI_gym │ │ ├── RL_brain.py │ │ ├── run_CartPole.py │ │ └── run_MountainCar.py │ ├── 7_Policy_gradient_softmax │ │ ├── RL_brain.py │ │ ├── run_CartPole.py │ │ └── run_MountainCar.py │ ├── 8_Actor_Critic_Advantage │ │ ├── AC_CartPole.py │ │ └── AC_continue_Pendulum.py │ └── 9_Deep_Deterministic_Policy_Gradient_DDPG │ │ ├── DDPG.py │ │ └── DDPG_update.py └── experiments │ ├── 2D_car │ ├── DDPG.py │ ├── car_env.py │ └── collision.py │ ├── Robot_arm │ ├── A3C.py │ ├── DDPG.py │ ├── DPPO.py │ └── arm_env.py │ ├── Solve_BipedalWalker │ ├── A3C.py │ ├── A3C_rnn.py │ ├── DDPG.py │ └── log │ │ └── events.out.tfevents.1490801027.Morvan │ └── Solve_LunarLander │ ├── A3C.py │ ├── DuelingDQNPrioritizedReplay.py │ └── run_LunarLander.py ├── Tensorflow-Project-Template ├── .gitignore ├── LICENSE ├── README.md ├── base │ ├── base_model.py │ └── base_train.py ├── configs │ └── example.json ├── data_loader │ └── data_generator.py ├── figures │ └── diagram.png ├── mains │ └── example.py ├── models │ ├── example_model.py │ └── template_model.py ├── trainers │ ├── example_trainer.py │ └── template_trainer.py └── utils │ ├── __init__.py │ ├── config.py │ ├── dirs.py │ ├── logger.py │ └── utils.py ├── cnn-text-classification-tf ├── data │ └── rt-polaritydata │ │ ├── rt-polarity.neg │ │ └── rt-polarity.pos ├── data_helpers.py ├── eval.py ├── text_cnn.py └── train.py ├── deep-learning-with-python-notebooks ├── 2.1-a-first-look-at-a-neural-network.ipynb ├── 3.5-classifying-movie-reviews.ipynb ├── 3.6-classifying-newswires.ipynb ├── 3.7-predicting-house-prices.ipynb ├── 4.4-overfitting-and-underfitting.ipynb ├── 5.1-introduction-to-convnets.ipynb ├── 5.2-using-convnets-with-small-datasets.ipynb ├── 5.3-using-a-pretrained-convnet.ipynb ├── 5.4-visualizing-what-convnets-learn.ipynb ├── 6.1-one-hot-encoding-of-words-or-characters.ipynb ├── 6.1-using-word-embeddings.ipynb ├── 6.2-understanding-recurrent-neural-networks.ipynb ├── 6.3-advanced-usage-of-recurrent-neural-networks.ipynb ├── 6.4-sequence-processing-with-convnets.ipynb ├── 8.1-text-generation-with-lstm.ipynb ├── 8.2-deep-dream.ipynb ├── 8.3-neural-style-transfer.ipynb ├── 8.4-generating-images-with-vaes.ipynb ├── 8.5-introduction-to-gans.ipynb ├── LICENSE └── README.md ├── deep_cnn ├── __init__.py ├── alexnet │ ├── BUILD │ ├── __init__.py │ └── alexnet_benchmark.py ├── cifar10 │ ├── BUILD │ ├── README.md │ ├── __init__.py │ ├── cifar10.py │ ├── cifar10_eval.py │ ├── cifar10_input.py │ ├── cifar10_input_test.py │ ├── cifar10_multi_gpu_train.py │ └── cifar10_train.py ├── cnn_layers_demo.py ├── imagenet │ ├── BUILD │ └── classify_image.py └── mnist │ ├── BUILD │ ├── __init__.py │ └── convolutional.py ├── distributed_dssm ├── .gitignore ├── 400-120-relu-50N.png ├── README.md ├── dist │ ├── dist-dssm-time-tf.png │ ├── dist-plot.py │ ├── dist-tf.png │ ├── dist_dssm_tf.sh │ └── sync-dssm-dist.py ├── single │ ├── dssm-denser.py │ ├── dssm.py │ ├── dssm_v2.py │ ├── dssm_v3.py │ └── plot.py └── tf-datasize.png ├── distributed_tensorflow ├── Distributed TensorFlow - A Gentle Introduction.ipynb ├── distributed-tensorflow-1.py ├── distributed_tensorflow-2.py ├── distributed_tensorflow_async1.py ├── distributed_tensorflow_sync1.py ├── mnist_dist.py ├── mnist_multi_ gpu_train.py └── tensorflow-mnist-multi-gpu.py ├── estimators ├── abalone.py ├── blog_custom_estimators.py └── cifar10_estimator │ ├── README.md │ ├── __init__.py │ ├── cifar10.py │ ├── cifar10_main.py │ ├── cifar10_model.py │ ├── cifar10_utils.py │ ├── cmle_config.yaml │ ├── generate_cifar10_tfrecords.py │ └── model_base.py ├── google-seq2seq ├── .coveragerc ├── .gitignore ├── .style.yapf ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin │ ├── __init__.py │ ├── data │ │ ├── cnn_daily_mail_summarization │ │ │ ├── process_data.sh │ │ │ └── process_story.py │ │ ├── toy.sh │ │ └── wmt16_en_de.sh │ ├── infer.py │ ├── tools │ │ ├── beam_search_viz │ │ │ ├── tree.css │ │ │ └── tree.js │ │ ├── generate_beam_viz.py │ │ ├── generate_toy_data.py │ │ ├── generate_vocab.py │ │ ├── multi-bleu.perl │ │ └── profile.py │ └── train.py ├── circle.yml ├── docs │ ├── concepts.md │ ├── contributing.md │ ├── data.md │ ├── decoders.md │ ├── encoders.md │ ├── extra.css │ ├── getting_started.md │ ├── help.md │ ├── image_captioning.md │ ├── images │ │ ├── nmt_tutorial_bleu.png │ │ └── nmt_tutorial_ppl.png │ ├── index.md │ ├── inference.md │ ├── license.md │ ├── models.md │ ├── nmt.md │ ├── results.md │ ├── summarization.md │ ├── tools.md │ └── training.md ├── example_configs │ ├── nmt_conv.yml │ ├── nmt_conv_small.yml │ ├── nmt_large.yml │ ├── nmt_medium.yml │ ├── nmt_small.yml │ ├── text_metrics_bpe.yml │ ├── text_metrics_sp.yml │ ├── train_seq2seq.yml │ └── train_seq2seq_delay_start.yml ├── mkdocs.yml ├── pylintrc ├── seq2seq │ ├── __init__.py │ ├── configurable.py │ ├── contrib │ │ ├── __init__.py │ │ ├── experiment.py │ │ ├── rnn_cell.py │ │ └── seq2seq │ │ │ ├── __init__.py │ │ │ ├── decoder.py │ │ │ └── helper.py │ ├── data │ │ ├── __init__.py │ │ ├── input_pipeline.py │ │ ├── parallel_data_provider.py │ │ ├── postproc.py │ │ ├── sequence_example_decoder.py │ │ ├── split_tokens_decoder.py │ │ └── vocab.py │ ├── decoders │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── attention_decoder.py │ │ ├── basic_decoder.py │ │ ├── beam_search_decoder.py │ │ └── rnn_decoder.py │ ├── encoders │ │ ├── __init__.py │ │ ├── conv_encoder.py │ │ ├── encoder.py │ │ ├── image_encoder.py │ │ ├── pooling_encoder.py │ │ └── rnn_encoder.py │ ├── global_vars.py │ ├── graph_module.py │ ├── graph_utils.py │ ├── inference │ │ ├── __init__.py │ │ ├── beam_search.py │ │ └── inference.py │ ├── losses.py │ ├── metrics │ │ ├── __init__.py │ │ ├── bleu.py │ │ ├── metric_specs.py │ │ └── rouge.py │ ├── models │ │ ├── __init__.py │ │ ├── attention_seq2seq.py │ │ ├── basic_seq2seq.py │ │ ├── bridges.py │ │ ├── image2seq.py │ │ ├── model_base.py │ │ └── seq2seq_model.py │ ├── tasks │ │ ├── __init__.py │ │ ├── decode_text.py │ │ ├── dump_attention.py │ │ ├── dump_beams.py │ │ └── inference_task.py │ ├── test │ │ ├── __init__.py │ │ ├── attention_test.py │ │ ├── beam_search_test.py │ │ ├── bridges_test.py │ │ ├── conv_encoder_test.py │ │ ├── data_test.py │ │ ├── decoder_test.py │ │ ├── example_config_test.py │ │ ├── hooks_test.py │ │ ├── input_pipeline_test.py │ │ ├── losses_test.py │ │ ├── metrics_test.py │ │ ├── models_test.py │ │ ├── pipeline_test.py │ │ ├── pooling_encoder_test.py │ │ ├── rnn_cell_test.py │ │ ├── rnn_encoder_test.py │ │ ├── train_utils_test.py │ │ ├── utils.py │ │ └── vocab_test.py │ └── training │ │ ├── __init__.py │ │ ├── hooks.py │ │ └── utils.py ├── setup.py └── tox.ini ├── image_tfrecords ├── create_tfrecord.py └── dataset_utils.py ├── kaggle-avazu-master ├── AvazuModelDocumentation.pdf ├── LICENSE.txt ├── READ.me ├── _0_run_me.sh ├── _1_encode_cat_features.py ├── _2b_generate_dataset_for_vw_fm.py ├── _2c_generate_fm_features.py ├── _3a_rf.py ├── _3b_gbdt.py ├── _3c_vw.py ├── _3d_fm.py ├── _4_post_processing.py └── utils.py ├── my_test_serving ├── remote_copy.sh ├── simple_math1_client.py └── simple_math1_model.py ├── nlp_estimator_tutorial ├── .gitignore ├── LICENSE ├── README.md ├── blog.md ├── conv.png ├── embeddings.gif ├── embeddings.png ├── loss.png ├── mermaid │ ├── cnn.mmd │ ├── cnn.mmd.svg │ ├── cnn.png │ ├── lstm.mmd │ ├── lstm.mmd.svg │ └── lstm.png ├── nlp_estimators.ipynb ├── nlp_estimators.py ├── pr.png ├── pr_curves.png └── token_weights.png ├── nmt ├── CONTRIBUTING.md ├── LICENSE ├── README.md └── nmt │ ├── .gitignore │ ├── __init__.py │ ├── attention_model.py │ ├── g3doc │ └── img │ │ ├── attention_equation_0.jpg │ │ ├── attention_equation_1.jpg │ │ ├── attention_mechanism.jpg │ │ ├── attention_vis.jpg │ │ ├── encdec.jpg │ │ ├── greedy_dec.jpg │ │ └── seq2seq.jpg │ ├── gnmt_model.py │ ├── inference.py │ ├── inference_test.py │ ├── model.py │ ├── model_helper.py │ ├── model_test.py │ ├── nmt.py │ ├── scripts │ ├── __init__.py │ ├── bleu.py │ ├── download_iwslt15.sh │ ├── rouge.py │ └── wmt16_en_de.sh │ ├── standard_hparams │ ├── iwslt15.json │ ├── wmt16.json │ ├── wmt16_gnmt_4_layer.json │ └── wmt16_gnmt_8_layer.json │ ├── testdata │ ├── deen_output │ ├── deen_ref_bpe │ ├── deen_ref_spm │ ├── label_ref │ ├── pred_output │ ├── test_embed.txt │ ├── test_embed_with_header.txt │ ├── test_infer_file │ ├── test_infer_vocab.src │ └── test_infer_vocab.tgt │ ├── train.py │ └── utils │ ├── __init__.py │ ├── common_test_utils.py │ ├── evaluation_utils.py │ ├── evaluation_utils_test.py │ ├── iterator_utils.py │ ├── iterator_utils_test.py │ ├── misc_utils.py │ ├── misc_utils_test.py │ ├── nmt_utils.py │ ├── vocab_utils.py │ └── vocab_utils_test.py ├── snippets ├── 03_implementing_tf_idf.py ├── Demo_Estimators_DataSet.py ├── Example of Seq2Seq with Attention using all the latest APIs.py ├── Example using TensorFlow Estimator, Experiment & Dataset.py ├── LabelImage.java ├── TFRecordOutputFormatStreaming.java ├── TensorFlow DNN Regressor with feature engineering.py ├── Wide_Multi-GPU_Network_with_Sparse_Matrix.py ├── between-graph_replication_async_train.py ├── dnn_linear_combined.py ├── fast_solution.py ├── google_retrain.py ├── learning2rank_metrics.py ├── lstm_seq2seq.py ├── nlp_estimators.py ├── parsing_ops_test.py ├── rank_metrics.py ├── triplet_movielens.py └── use_pretrain_inception_model.py ├── speech_commands_examples ├── freeze.py ├── models.py ├── models_test.py └── train.py ├── tensorboard ├── demo-embedding1.py ├── demo_embedding2.py ├── mnist_t-sne.py └── tf-dev-summit-tensorboard-tutorial.py ├── tensorflow-DeepFM ├── .gitignore ├── DeepFM.py ├── LICENSE ├── README.md └── example │ ├── DataReader.py │ ├── README.md │ ├── __init__.py │ ├── config.py │ ├── data │ └── README.md │ ├── main.py │ ├── metrics.py │ └── output │ └── README.md ├── tensorflow_template_application ├── .gitignore ├── LICENSE ├── README.md ├── cpp_predict_client │ ├── README.md │ ├── generate_proto_files │ │ ├── generate_proto_files.sh │ │ ├── model.proto │ │ ├── predict.proto │ │ ├── prediction_service.proto │ │ └── tensorflow │ │ │ ├── contrib │ │ │ ├── ios_examples │ │ │ │ └── .gitignore │ │ │ ├── makefile │ │ │ │ └── .gitignore │ │ │ ├── pi_examples │ │ │ │ └── .gitignore │ │ │ ├── session_bundle │ │ │ │ └── manifest.proto │ │ │ └── tensorboard │ │ │ │ └── plugins │ │ │ │ └── projector │ │ │ │ └── projector_config.proto │ │ │ ├── core │ │ │ ├── debug │ │ │ │ └── debug_service.proto │ │ │ ├── example │ │ │ │ ├── example.proto │ │ │ │ ├── example_parser_configuration.proto │ │ │ │ └── feature.proto │ │ │ ├── framework │ │ │ │ ├── allocation_description.proto │ │ │ │ ├── attr_value.proto │ │ │ │ ├── cost_graph.proto │ │ │ │ ├── device_attributes.proto │ │ │ │ ├── function.proto │ │ │ │ ├── graph.proto │ │ │ │ ├── kernel_def.proto │ │ │ │ ├── log_memory.proto │ │ │ │ ├── node_def.proto │ │ │ │ ├── op_def.proto │ │ │ │ ├── resource_handle.proto │ │ │ │ ├── step_stats.proto │ │ │ │ ├── summary.proto │ │ │ │ ├── tensor.proto │ │ │ │ ├── tensor_description.proto │ │ │ │ ├── tensor_shape.proto │ │ │ │ ├── tensor_slice.proto │ │ │ │ ├── types.proto │ │ │ │ ├── variable.proto │ │ │ │ └── versions.proto │ │ │ ├── kernels │ │ │ │ └── reader_base.proto │ │ │ ├── protobuf │ │ │ │ ├── config.proto │ │ │ │ ├── control_flow.proto │ │ │ │ ├── master.proto │ │ │ │ ├── master_service.proto │ │ │ │ ├── meta_graph.proto │ │ │ │ ├── named_tensor.proto │ │ │ │ ├── queue_runner.proto │ │ │ │ ├── saved_model.proto │ │ │ │ ├── saver.proto │ │ │ │ ├── tensor_bundle.proto │ │ │ │ ├── tensorflow_server.proto │ │ │ │ ├── worker.proto │ │ │ │ └── worker_service.proto │ │ │ └── util │ │ │ │ ├── event.proto │ │ │ │ ├── memmapped_file_system.proto │ │ │ │ ├── saved_tensor_slice.proto │ │ │ │ └── test_log.proto │ │ │ ├── examples │ │ │ └── udacity │ │ │ │ └── .gitignore │ │ │ ├── go │ │ │ ├── genop │ │ │ │ └── .gitignore │ │ │ └── op │ │ │ │ └── .gitignore │ │ │ ├── python │ │ │ ├── framework │ │ │ │ └── cpp_shape_inference.proto │ │ │ ├── training │ │ │ │ └── checkpoint_state.proto │ │ │ └── util │ │ │ │ └── protobuf │ │ │ │ └── compare_test.proto │ │ │ ├── tensorboard │ │ │ ├── .bowerrc │ │ │ └── .gitignore │ │ │ └── tools │ │ │ ├── ci_build │ │ │ └── install │ │ │ │ └── .bazelrc │ │ │ ├── git │ │ │ └── .gitignore │ │ │ ├── proto_text │ │ │ └── test.proto │ │ │ ├── swig │ │ │ └── .gitignore │ │ │ └── tfprof │ │ │ ├── tfprof_log.proto │ │ │ ├── tfprof_options.proto │ │ │ └── tfprof_output.proto │ └── sparse_predict_client.cc ├── cpp_predict_server │ └── README.md ├── data │ ├── README.md │ ├── a8a │ │ ├── a8a_test.libsvm │ │ ├── a8a_test.libsvm.tfrecords │ │ ├── a8a_train.libsvm │ │ ├── a8a_train.libsvm.tfrecords │ │ ├── generate_libsvm_tfrecord.py │ │ └── print_libsvm_tfrecords.py │ ├── a8a_test.libsvm │ ├── boston_housing │ │ ├── README.md │ │ ├── generate_csv_tfrecords.py │ │ ├── print_csv_tfrecords.py │ │ ├── train.csv │ │ └── train.csv.tfrecords │ ├── cancer │ │ ├── cancer_test.csv │ │ ├── cancer_test.csv.tfrecords │ │ ├── cancer_train.csv │ │ ├── cancer_train.csv.tfrecords │ │ ├── generate_csv_tfrecords.py │ │ └── print_csv_tfrecords.py │ ├── cancer_test.csv │ ├── iris │ │ ├── download_iris.py │ │ ├── generate_csv_tfrecords.py │ │ ├── iris_test.csv │ │ ├── iris_test.csv.tfrecords │ │ ├── iris_train.csv │ │ ├── iris_train.csv.tfrecords │ │ └── print_csv_tfrecords.py │ └── lung │ │ ├── convert_dcm_to_csv.py │ │ ├── fa7a21165ae152b13def786e6afc3edf.dcm │ │ ├── fa7a21165ae152b13def786e6afc3edf.dcm.csv │ │ ├── fa7a21165ae152b13def786e6afc3edf.dcm.csv.tfrecords │ │ ├── generate_csv_tfrecords.py │ │ ├── print_csv_tfrecords.py │ │ └── raw_data │ │ └── stage1_labels.csv ├── dense_classifier.py ├── distributed │ ├── README.md │ ├── checkpoint │ │ └── .gitkeeper │ └── dense_classifier.py ├── golang_predict_client │ ├── README.md │ └── src │ │ ├── generate_golang_files │ │ ├── model.proto │ │ ├── predict.proto │ │ ├── prediction_service.proto │ │ └── tensorflow │ │ │ ├── contrib │ │ │ ├── ios_examples │ │ │ │ └── .gitignore │ │ │ ├── makefile │ │ │ │ └── .gitignore │ │ │ ├── pi_examples │ │ │ │ └── .gitignore │ │ │ ├── session_bundle │ │ │ │ └── manifest.proto │ │ │ └── tensorboard │ │ │ │ └── plugins │ │ │ │ └── projector │ │ │ │ └── projector_config.proto │ │ │ ├── core │ │ │ ├── debug │ │ │ │ └── debug_service.proto │ │ │ ├── example │ │ │ │ ├── example.proto │ │ │ │ ├── example_parser_configuration.proto │ │ │ │ └── feature.proto │ │ │ ├── framework │ │ │ │ ├── allocation_description.proto │ │ │ │ ├── attr_value.proto │ │ │ │ ├── cost_graph.proto │ │ │ │ ├── device_attributes.proto │ │ │ │ ├── function.proto │ │ │ │ ├── graph.proto │ │ │ │ ├── kernel_def.proto │ │ │ │ ├── log_memory.proto │ │ │ │ ├── node_def.proto │ │ │ │ ├── op_def.proto │ │ │ │ ├── resource_handle.proto │ │ │ │ ├── step_stats.proto │ │ │ │ ├── summary.proto │ │ │ │ ├── tensor.proto │ │ │ │ ├── tensor_description.proto │ │ │ │ ├── tensor_shape.proto │ │ │ │ ├── tensor_slice.proto │ │ │ │ ├── types.proto │ │ │ │ ├── variable.proto │ │ │ │ └── versions.proto │ │ │ ├── kernels │ │ │ │ └── reader_base.proto │ │ │ ├── protobuf │ │ │ │ ├── config.proto │ │ │ │ ├── control_flow.proto │ │ │ │ ├── master.proto │ │ │ │ ├── master_service.proto │ │ │ │ ├── meta_graph.proto │ │ │ │ ├── named_tensor.proto │ │ │ │ ├── queue_runner.proto │ │ │ │ ├── saved_model.proto │ │ │ │ ├── saver.proto │ │ │ │ ├── tensor_bundle.proto │ │ │ │ ├── tensorflow_server.proto │ │ │ │ ├── worker.proto │ │ │ │ └── worker_service.proto │ │ │ └── util │ │ │ │ ├── event.proto │ │ │ │ ├── memmapped_file_system.proto │ │ │ │ ├── saved_tensor_slice.proto │ │ │ │ └── test_log.proto │ │ │ ├── examples │ │ │ └── udacity │ │ │ │ └── .gitignore │ │ │ ├── go │ │ │ ├── genop │ │ │ │ └── .gitignore │ │ │ └── op │ │ │ │ └── .gitignore │ │ │ ├── python │ │ │ ├── framework │ │ │ │ └── cpp_shape_inference.proto │ │ │ ├── training │ │ │ │ └── checkpoint_state.proto │ │ │ └── util │ │ │ │ └── protobuf │ │ │ │ └── compare_test.proto │ │ │ ├── tensorboard │ │ │ ├── .bowerrc │ │ │ └── .gitignore │ │ │ └── tools │ │ │ ├── ci_build │ │ │ └── install │ │ │ │ └── .bazelrc │ │ │ ├── git │ │ │ └── .gitignore │ │ │ ├── proto_text │ │ │ └── test.proto │ │ │ ├── swig │ │ │ └── .gitignore │ │ │ └── tfprof │ │ │ ├── tfprof_log.proto │ │ │ ├── tfprof_options.proto │ │ │ └── tfprof_output.proto │ │ ├── generate_proto_files.sh │ │ ├── predict_client.go │ │ └── util.go ├── http_service │ ├── README.md │ ├── cancer_predict │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── db.sqlite3 │ ├── images │ │ ├── online_train.png │ │ └── predict.png │ ├── manage.py │ ├── predict.sh │ ├── predict_examples.json │ └── restful_server │ │ ├── __init__.py │ │ ├── settings.py │ │ ├── urls.py │ │ └── wsgi.py ├── java_predict_client │ ├── README.md │ ├── pom.xml │ ├── run │ ├── src │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── tobe │ │ │ │ └── client │ │ │ │ ├── DensePredictClient.java │ │ │ │ └── SparsePredictClient.java │ │ │ ├── proto │ │ │ ├── model.proto │ │ │ ├── predict.proto │ │ │ ├── prediction_service.proto │ │ │ └── tensorflow │ │ │ │ ├── contrib │ │ │ │ ├── ios_examples │ │ │ │ │ └── .gitignore │ │ │ │ ├── makefile │ │ │ │ │ └── .gitignore │ │ │ │ ├── pi_examples │ │ │ │ │ └── .gitignore │ │ │ │ ├── session_bundle │ │ │ │ │ └── manifest.proto │ │ │ │ └── tensorboard │ │ │ │ │ └── plugins │ │ │ │ │ └── projector │ │ │ │ │ └── projector_config.proto │ │ │ │ ├── core │ │ │ │ ├── debug │ │ │ │ │ └── debug_service.proto │ │ │ │ ├── example │ │ │ │ │ ├── example.proto │ │ │ │ │ ├── example_parser_configuration.proto │ │ │ │ │ └── feature.proto │ │ │ │ ├── framework │ │ │ │ │ ├── allocation_description.proto │ │ │ │ │ ├── attr_value.proto │ │ │ │ │ ├── cost_graph.proto │ │ │ │ │ ├── device_attributes.proto │ │ │ │ │ ├── function.proto │ │ │ │ │ ├── graph.proto │ │ │ │ │ ├── kernel_def.proto │ │ │ │ │ ├── log_memory.proto │ │ │ │ │ ├── node_def.proto │ │ │ │ │ ├── op_def.proto │ │ │ │ │ ├── resource_handle.proto │ │ │ │ │ ├── step_stats.proto │ │ │ │ │ ├── summary.proto │ │ │ │ │ ├── tensor.proto │ │ │ │ │ ├── tensor_description.proto │ │ │ │ │ ├── tensor_shape.proto │ │ │ │ │ ├── tensor_slice.proto │ │ │ │ │ ├── types.proto │ │ │ │ │ ├── variable.proto │ │ │ │ │ └── versions.proto │ │ │ │ ├── kernels │ │ │ │ │ └── reader_base.proto │ │ │ │ ├── protobuf │ │ │ │ │ ├── config.proto │ │ │ │ │ ├── control_flow.proto │ │ │ │ │ ├── master.proto │ │ │ │ │ ├── master_service.proto │ │ │ │ │ ├── meta_graph.proto │ │ │ │ │ ├── named_tensor.proto │ │ │ │ │ ├── queue_runner.proto │ │ │ │ │ ├── saved_model.proto │ │ │ │ │ ├── saver.proto │ │ │ │ │ ├── tensor_bundle.proto │ │ │ │ │ ├── tensorflow_server.proto │ │ │ │ │ ├── worker.proto │ │ │ │ │ └── worker_service.proto │ │ │ │ └── util │ │ │ │ │ ├── event.proto │ │ │ │ │ ├── memmapped_file_system.proto │ │ │ │ │ ├── saved_tensor_slice.proto │ │ │ │ │ └── test_log.proto │ │ │ │ ├── examples │ │ │ │ └── udacity │ │ │ │ │ └── .gitignore │ │ │ │ ├── go │ │ │ │ ├── genop │ │ │ │ │ └── .gitignore │ │ │ │ └── op │ │ │ │ │ └── .gitignore │ │ │ │ ├── python │ │ │ │ ├── framework │ │ │ │ │ └── cpp_shape_inference.proto │ │ │ │ ├── training │ │ │ │ │ └── checkpoint_state.proto │ │ │ │ └── util │ │ │ │ │ └── protobuf │ │ │ │ │ └── compare_test.proto │ │ │ │ ├── tensorboard │ │ │ │ ├── .bowerrc │ │ │ │ └── .gitignore │ │ │ │ └── tools │ │ │ │ ├── ci_build │ │ │ │ └── install │ │ │ │ │ └── .bazelrc │ │ │ │ ├── git │ │ │ │ └── .gitignore │ │ │ │ ├── proto_text │ │ │ │ └── test.proto │ │ │ │ ├── swig │ │ │ │ └── .gitignore │ │ │ │ └── tfprof │ │ │ │ ├── tfprof_log.proto │ │ │ │ ├── tfprof_options.proto │ │ │ │ └── tfprof_output.proto │ │ │ └── scala │ │ │ └── com │ │ │ └── tobe │ │ │ ├── client │ │ │ ├── ScalaDensePredictClient.scala │ │ │ └── SparkDensePredictClient.scala │ │ │ └── data │ │ │ ├── GenerateDenseTfrecords.scala │ │ │ └── GenerateSparseTfrecords.scala │ └── update_tensorflow_proto_files.sh ├── minimal_model │ ├── README.md │ ├── benchmark_predict.py │ ├── data.json │ ├── model │ │ └── 00000001 │ │ │ ├── checkpoint │ │ │ ├── export.data-00000-of-00001 │ │ │ ├── export.index │ │ │ └── export.meta │ ├── python_predict_client │ │ ├── benchmark_latency.py │ │ ├── benchmark_qps.py │ │ ├── model_pb2.py │ │ ├── predict_client.py │ │ ├── predict_pb2.py │ │ └── prediction_service_pb2.py │ └── train.py ├── python_predict_client │ ├── README.md │ ├── generate_python_files │ │ ├── generate_proto_files.sh │ │ ├── model.proto │ │ ├── predict.proto │ │ ├── prediction_service.proto │ │ └── tensorflow │ │ │ ├── contrib │ │ │ ├── ios_examples │ │ │ │ └── .gitignore │ │ │ ├── makefile │ │ │ │ └── .gitignore │ │ │ ├── pi_examples │ │ │ │ └── .gitignore │ │ │ ├── session_bundle │ │ │ │ └── manifest.proto │ │ │ └── tensorboard │ │ │ │ └── plugins │ │ │ │ └── projector │ │ │ │ └── projector_config.proto │ │ │ ├── core │ │ │ ├── debug │ │ │ │ └── debug_service.proto │ │ │ ├── example │ │ │ │ ├── example.proto │ │ │ │ ├── example_parser_configuration.proto │ │ │ │ └── feature.proto │ │ │ ├── framework │ │ │ │ ├── allocation_description.proto │ │ │ │ ├── attr_value.proto │ │ │ │ ├── cost_graph.proto │ │ │ │ ├── device_attributes.proto │ │ │ │ ├── function.proto │ │ │ │ ├── graph.proto │ │ │ │ ├── kernel_def.proto │ │ │ │ ├── log_memory.proto │ │ │ │ ├── node_def.proto │ │ │ │ ├── op_def.proto │ │ │ │ ├── resource_handle.proto │ │ │ │ ├── step_stats.proto │ │ │ │ ├── summary.proto │ │ │ │ ├── tensor.proto │ │ │ │ ├── tensor_description.proto │ │ │ │ ├── tensor_shape.proto │ │ │ │ ├── tensor_slice.proto │ │ │ │ ├── types.proto │ │ │ │ ├── variable.proto │ │ │ │ └── versions.proto │ │ │ ├── kernels │ │ │ │ └── reader_base.proto │ │ │ ├── protobuf │ │ │ │ ├── config.proto │ │ │ │ ├── control_flow.proto │ │ │ │ ├── master.proto │ │ │ │ ├── master_service.proto │ │ │ │ ├── meta_graph.proto │ │ │ │ ├── named_tensor.proto │ │ │ │ ├── queue_runner.proto │ │ │ │ ├── saved_model.proto │ │ │ │ ├── saver.proto │ │ │ │ ├── tensor_bundle.proto │ │ │ │ ├── tensorflow_server.proto │ │ │ │ ├── worker.proto │ │ │ │ └── worker_service.proto │ │ │ └── util │ │ │ │ ├── event.proto │ │ │ │ ├── memmapped_file_system.proto │ │ │ │ ├── saved_tensor_slice.proto │ │ │ │ └── test_log.proto │ │ │ ├── examples │ │ │ └── udacity │ │ │ │ └── .gitignore │ │ │ ├── go │ │ │ ├── genop │ │ │ │ └── .gitignore │ │ │ └── op │ │ │ │ └── .gitignore │ │ │ ├── python │ │ │ ├── framework │ │ │ │ └── cpp_shape_inference.proto │ │ │ ├── training │ │ │ │ └── checkpoint_state.proto │ │ │ └── util │ │ │ │ └── protobuf │ │ │ │ └── compare_test.proto │ │ │ ├── tensorboard │ │ │ ├── .bowerrc │ │ │ └── .gitignore │ │ │ └── tools │ │ │ ├── ci_build │ │ │ └── install │ │ │ │ └── .bazelrc │ │ │ ├── git │ │ │ └── .gitignore │ │ │ ├── proto_text │ │ │ └── test.proto │ │ │ ├── swig │ │ │ └── .gitignore │ │ │ └── tfprof │ │ │ ├── tfprof_log.proto │ │ │ ├── tfprof_options.proto │ │ │ └── tfprof_output.proto │ ├── model_pb2.py │ ├── predict_client.py │ ├── predict_pb2.py │ ├── prediction_service_pb2.py │ ├── requirements.txt │ └── sparse_predict_client.py ├── requirements.txt ├── setup.py ├── sklearn_exmaples │ └── cancer_classifier.py ├── sparse_classifier.py └── trainer │ └── __init__.py ├── tensorflow_transfer_cifar10 ├── data_utils.py ├── extract.py ├── transfer_cifar10_softmax.py └── tsne.py ├── tf-bpr ├── Advanced BPR.ipynb ├── README.md ├── corpus.py ├── deploy_bpr.sh ├── models │ ├── __init__.py │ ├── bpr.py │ ├── hbpr.py │ ├── model.py │ └── vbpr.py ├── sampling.py └── train.py ├── tf-serving-demo ├── model │ ├── model.py │ └── three_x_plus_y │ │ └── 1 │ │ ├── saved_model.pb │ │ └── variables │ │ ├── variables.data-00000-of-00001 │ │ └── variables.index └── src │ └── main │ ├── proto │ ├── tensorflow │ │ └── core │ │ │ ├── example │ │ │ ├── example.proto │ │ │ └── feature.proto │ │ │ ├── framework │ │ │ ├── attr_value.proto │ │ │ ├── function.proto │ │ │ ├── graph.proto │ │ │ ├── node_def.proto │ │ │ ├── op_def.proto │ │ │ ├── resource_handle.proto │ │ │ ├── tensor.proto │ │ │ ├── tensor_shape.proto │ │ │ ├── types.proto │ │ │ └── versions.proto │ │ │ └── protobuf │ │ │ ├── meta_graph.proto │ │ │ └── saver.proto │ └── tensorflow_serving │ │ └── apis │ │ ├── classification.proto │ │ ├── get_model_metadata.proto │ │ ├── input.proto │ │ ├── model.proto │ │ ├── predict.proto │ │ ├── prediction_service.proto │ │ └── regression.proto │ └── scala │ └── com │ └── fdahms │ └── tfservingdemo │ └── Main.scala ├── tf_deep_ctr ├── README.md └── deep_ctr │ ├── Feature_pipeline │ ├── get_aliccp_tfrecord.py │ ├── get_criteo_feature.py │ ├── get_frape_feature.py │ └── get_smart_feature.py │ ├── Model_pipeline │ ├── AFM.py │ ├── DCN.py │ ├── DIN.py │ ├── DeepFM.py │ ├── DeepMVM.py │ ├── NFM.py │ ├── PNN.py │ └── wide_n_deep.py │ ├── README.md │ ├── Serving_pipeline │ ├── deep_fm_serving_client.cpp │ ├── deep_fm_serving_client.h │ ├── wide_n_deep_serving_client.cpp │ └── wide_n_deep_serving_client.h │ ├── run.sh │ └── run_dist.sh ├── tf_import_data ├── convert_to_records.py ├── fully_connected_reader.py └── test_libsvm_tfrecords.py ├── tf_serving ├── example1_client.py ├── example1_save_model.py ├── exporter_test.py ├── inception_client.cc ├── inception_saved_model.py ├── mnist_client.py ├── mnist_saved_model.py └── saved_model_half_plus_two.py ├── tf_wide_n_deep ├── census_dataset.py ├── census_main.py ├── census_test.csv ├── census_test.py ├── dnn.py ├── dnn_linear_combined.py ├── estimator.py ├── feature_column.py ├── linear.py ├── logger.py ├── metric_hook.py ├── training.py ├── wide_deep_run_loop.py └── wide_n_deep_tutorial.py └── transfer_learning_tutorial ├── eval_flowers.py ├── inception_preprocessing.py ├── inception_resnet_v2.py └── train_flowers.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.pyc 3 | *.libsvm 4 | .ipynb_checkpoints 5 | *.png -------------------------------------------------------------------------------- /CTR_NN/README.md: -------------------------------------------------------------------------------- 1 | # CTR_NN 2 | CTR prediction using Neural Network methods
3 | 基于深度学习的CTR预估，从FM推演各深度学习CTR预估模型（附代码） 4 | 5 | 详情请参见博文[《从FM推演各深度学习CTR预估模型（附代码）》](https://blog.csdn.net/han_xiaoyang/article/details/81031961)
6 | 部分代码参考[lambdaji](https://github.com/lambdaji)同学，对此表示感谢 7 | 8 | 代码所需数据可在ctr_data.txt中的网盘地址中下载到。 9 | -------------------------------------------------------------------------------- /CTR_NN/ctr_data.txt: -------------------------------------------------------------------------------- 1 | 链接: https://pan.baidu.com/s/1tIhlbWaVHbKoLd4XAwrUDw 密码: veww 2 | -------------------------------------------------------------------------------- /CTR_NN/从FM推演各深度学习CTR预估模型（附代码）.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/CTR_NN/从FM推演各深度学习CTR预估模型（附代码）.pdf -------------------------------------------------------------------------------- /DeepRL-Agents/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Arthur Juliani 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /DeepRL-Agents/basic.wad: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/DeepRL-Agents/basic.wad -------------------------------------------------------------------------------- /Hvass-Labs_TensorFlow-Tutorials/forks.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Tutorials - Forks 2 | 3 | These are forks of the [original TensorFlow Tutorials by Hvass-Labs](https://github.com/Hvass-Labs/TensorFlow-Tutorials). 4 | They are not developed or even reviewed by the original author, who takes no reponsibility for these forks. 5 | 6 | If you have made a fork of the TensorFlow Tutorials with substantial modifications that you feel may be useful to others, 7 | then please [open a new issue on GitHub](https://github.com/Hvass-Labs/TensorFlow-Tutorials/issues) with a link and short description. 8 | 9 | * [Keras port of some tutorials.](https://github.com/chidochipotle/TensorFlow-Tutorials) 10 | * [The Inception model as an OpenFaaS function.](https://github.com/faas-and-furious/inception-function) 11 | -------------------------------------------------------------------------------- /Learning-PySpark/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /Learning-PySpark/.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/additionalCode/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='PySparkUtilities', 5 | version='0.1dev', 6 | packages=['utilities', 'utilities/converters'], 7 | license=''' 8 | Creative Commons 9 | Attribution-Noncommercial-Share Alike license''', 10 | long_description=''' 11 | An example of how to package code for PySpark''' 12 | ) -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/additionalCode/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | from .geoCalc import geoCalc 2 | 3 | __all__ = ['geoCalc','converters'] -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/additionalCode/utilities/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | class BaseConverter(metaclass=ABCMeta): 4 | @staticmethod 5 | @abstractmethod 6 | def convert(f, t): 7 | raise NotImplementedError 8 | 9 | if __name__ == '__main__': 10 | i = BaseConverter() -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/additionalCode/utilities/converters/__init__.py: -------------------------------------------------------------------------------- 1 | from .distance import metricImperial 2 | 3 | __all__ = ['metricImperial'] -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/additionalCode/utilities/converters/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | class BaseConverter(metaclass=ABCMeta): 4 | @staticmethod 5 | @abstractmethod 6 | def convert(f, t): 7 | raise NotImplementedError 8 | 9 | if __name__ == '__main__': 10 | i = BaseConverter() -------------------------------------------------------------------------------- /Learning-PySpark/Chapter10/launch_spark_submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | unset PYSPARK_DRIVER_PYTHON 3 | spark-submit $* 4 | export PYSPARK_DRIVER_PYTHON=jupyter -------------------------------------------------------------------------------- /Learning-PySpark/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DataScienceCodes 2 | codes collection for ML/DL/RL/BigData projects 3 | -------------------------------------------------------------------------------- /Reinforcement-learning-with-tensorflow-master/RL_cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/Reinforcement-learning-with-tensorflow-master/RL_cover.jpg -------------------------------------------------------------------------------- /Reinforcement-learning-with-tensorflow-master/experiments/Solve_BipedalWalker/log/events.out.tfevents.1490801027.Morvan: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/Reinforcement-learning-with-tensorflow-master/experiments/Solve_BipedalWalker/log/events.out.tfevents.1490801027.Morvan -------------------------------------------------------------------------------- /Tensorflow-Project-Template/configs/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "exp_name": "example", 3 | "num_epochs": 10, 4 | "num_iter_per_epoch": 10, 5 | "learning_rate": 0.001, 6 | "batch_size": 16, 7 | "state_size": [784] 8 | } -------------------------------------------------------------------------------- /Tensorflow-Project-Template/data_loader/data_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class DataGenerator(): 5 | def __init__(self, config): 6 | self.config = config 7 | # load data here 8 | self.input = np.ones((500, 784)) 9 | self.y = np.ones((500, 10)) 10 | def next_batch(self, batch_size): 11 | idx = np.random.choice(500, batch_size) 12 | yield self.input[idx], self.y[idx] 13 | -------------------------------------------------------------------------------- /Tensorflow-Project-Template/figures/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/Tensorflow-Project-Template/figures/diagram.png -------------------------------------------------------------------------------- /Tensorflow-Project-Template/models/template_model.py: -------------------------------------------------------------------------------- 1 | from base.base_model import BaseModel 2 | import tensorflow as tf 3 | 4 | 5 | class TemplateModel(BaseModel): 6 | def __init__(self, config): 7 | super(TemplateModel, self).__init__(config) 8 | 9 | self.build_model() 10 | self.init_saver() 11 | 12 | def build_model(self): 13 | # here you build the tensorflow graph of any model you want and also define the loss. 14 | pass 15 | 16 | def init_saver(self): 17 | #here you initalize the tensorflow saver that will be used in saving the checkpoints. 18 | # self.saver = tf.train.Saver(max_to_keep=self.config.max_to_keep) 19 | 20 | pass 21 | -------------------------------------------------------------------------------- /Tensorflow-Project-Template/trainers/template_trainer.py: -------------------------------------------------------------------------------- 1 | from base.base_train import BaseTrain 2 | from tqdm import tqdm 3 | import numpy as np 4 | 5 | 6 | class TemplateTrainer(BaseTrain): 7 | def __init__(self, sess, model, data, config, logger): 8 | super(TemplateTrainer, self).__init__(sess, model, data, config, logger) 9 | 10 | def train_epoch(self): 11 | """ 12 | implement the logic of epoch: 13 | -loop on the number of iterations in the config and call the train step 14 | -add any summaries you want using the summary 15 | """ 16 | pass 17 | 18 | def train_step(self): 19 | """ 20 | implement the logic of the train step 21 | - run the tensorflow session 22 | - return any metrics you need to summarize 23 | """ 24 | pass 25 | -------------------------------------------------------------------------------- /Tensorflow-Project-Template/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/Tensorflow-Project-Template/utils/__init__.py -------------------------------------------------------------------------------- /Tensorflow-Project-Template/utils/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | from bunch import Bunch 3 | import os 4 | 5 | 6 | def get_config_from_json(json_file): 7 | """ 8 | Get the config from a json file 9 | :param json_file: 10 | :return: config(namespace) or config(dictionary) 11 | """ 12 | # parse the configurations from the config json file provided 13 | with open(json_file, 'r') as config_file: 14 | config_dict = json.load(config_file) 15 | 16 | # convert the dictionary to a namespace using bunch lib 17 | config = Bunch(config_dict) 18 | 19 | return config, config_dict 20 | 21 | 22 | def process_config(jsonfile): 23 | config, _ = get_config_from_json(jsonfile) 24 | config.summary_dir = os.path.join("../experiments", config.exp_name, "summary") 25 | config.checkpoint_dir = os.path.join("../experiments", config.exp_name, "checkpoint") 26 | return config -------------------------------------------------------------------------------- /Tensorflow-Project-Template/utils/dirs.py: -------------------------------------------------------------------------------- 1 | import os 2 | def create_dirs(dirs): 3 | """ 4 | dirs - a list of directories to create if these directories are not found 5 | :param dirs: 6 | :return exit_code: 0:success -1:failed 7 | """ 8 | try: 9 | for dir_ in dirs: 10 | if not os.path.exists(dir_): 11 | os.makedirs(dir_) 12 | return 0 13 | except Exception as err: 14 | print("Creating directories error: {0}".format(err)) 15 | exit(-1) 16 | -------------------------------------------------------------------------------- /Tensorflow-Project-Template/utils/utils.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | def get_args(): 3 | argparser = argparse.ArgumentParser(description=__doc__) 4 | argparser.add_argument( 5 | '-c', '--config', 6 | metavar='C', 7 | default='None', 8 | help='The Configuration file') 9 | args = argparser.parse_args() 10 | return args -------------------------------------------------------------------------------- /deep-learning-with-python-notebooks/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 François Chollet 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /deep_cnn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/deep_cnn/__init__.py -------------------------------------------------------------------------------- /deep_cnn/alexnet/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Benchmark for AlexNet. 3 | 4 | licenses(["notice"]) # Apache 2.0 5 | 6 | exports_files(["LICENSE"]) 7 | 8 | py_binary( 9 | name = "alexnet_benchmark", 10 | srcs = [ 11 | "alexnet_benchmark.py", 12 | ], 13 | srcs_version = "PY2AND3", 14 | deps = [ 15 | "//tensorflow:tensorflow_py", 16 | ], 17 | ) 18 | 19 | filegroup( 20 | name = "all_files", 21 | srcs = glob( 22 | ["**/*"], 23 | exclude = [ 24 | "**/METADATA", 25 | "**/OWNERS", 26 | ], 27 | ), 28 | visibility = ["//tensorflow:__subpackages__"], 29 | ) 30 | -------------------------------------------------------------------------------- /deep_cnn/alexnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/deep_cnn/alexnet/__init__.py -------------------------------------------------------------------------------- /deep_cnn/cifar10/README.md: -------------------------------------------------------------------------------- 1 | CIFAR-10 is a common benchmark in machine learning for image recognition. 2 | 3 | http://www.cs.toronto.edu/~kriz/cifar.html 4 | 5 | Code in this directory demonstrates how to use TensorFlow to train and evaluate a convolutional neural network (CNN) on both CPU and GPU. We also demonstrate how to train a CNN over multiple GPUs. 6 | 7 | Detailed instructions on how to get started available at: 8 | 9 | http://tensorflow.org/tutorials/deep_cnn/ 10 | 11 | -------------------------------------------------------------------------------- /deep_cnn/cifar10/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Makes helper libraries available in the cifar10 package.""" 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import cifar10 22 | import cifar10_input 23 | -------------------------------------------------------------------------------- /deep_cnn/imagenet/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Example TensorFlow models for ImageNet. 3 | 4 | licenses(["notice"]) # Apache 2.0 5 | 6 | exports_files(["LICENSE"]) 7 | 8 | py_binary( 9 | name = "classify_image", 10 | srcs = [ 11 | "classify_image.py", 12 | ], 13 | srcs_version = "PY2AND3", 14 | visibility = ["//tensorflow:__subpackages__"], 15 | deps = [ 16 | "//tensorflow:tensorflow_py", 17 | ], 18 | ) 19 | 20 | filegroup( 21 | name = "all_files", 22 | srcs = glob( 23 | ["**/*"], 24 | exclude = [ 25 | "**/METADATA", 26 | "**/OWNERS", 27 | ], 28 | ), 29 | visibility = ["//tensorflow:__subpackages__"], 30 | ) 31 | -------------------------------------------------------------------------------- /deep_cnn/mnist/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Example TensorFlow models for MNIST that achieves high accuracy 3 | 4 | licenses(["notice"]) # Apache 2.0 5 | 6 | exports_files(["LICENSE"]) 7 | 8 | py_binary( 9 | name = "convolutional", 10 | srcs = [ 11 | "convolutional.py", 12 | ], 13 | srcs_version = "PY2AND3", 14 | visibility = ["//tensorflow:__subpackages__"], 15 | deps = ["//tensorflow:tensorflow_py"], 16 | ) 17 | 18 | py_test( 19 | name = "convolutional_test", 20 | size = "medium", 21 | srcs = [ 22 | "convolutional.py", 23 | ], 24 | args = [ 25 | "--self_test", 26 | ], 27 | main = "convolutional.py", 28 | srcs_version = "PY2AND3", 29 | deps = ["//tensorflow:tensorflow_py"], 30 | ) 31 | 32 | filegroup( 33 | name = "all_files", 34 | srcs = glob( 35 | ["**/*"], 36 | exclude = [ 37 | "**/METADATA", 38 | "**/OWNERS", 39 | ], 40 | ), 41 | visibility = ["//tensorflow:__subpackages__"], 42 | ) 43 | -------------------------------------------------------------------------------- /deep_cnn/mnist/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/deep_cnn/mnist/__init__.py -------------------------------------------------------------------------------- /distributed_dssm/.gitignore: -------------------------------------------------------------------------------- 1 | data/* 2 | -------------------------------------------------------------------------------- /distributed_dssm/400-120-relu-50N.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/distributed_dssm/400-120-relu-50N.png -------------------------------------------------------------------------------- /distributed_dssm/README.md: -------------------------------------------------------------------------------- 1 | # dssm on Tensorflow 2 | 3 | Document at http://liaha.github.io/models/2016/06/21/dssm-on-tensorflow.html 4 | 5 | Please refer to https://www.microsoft.com/en-us/research/project/dssm/ for more information. 6 | 7 | # Notice 8 | Thank everyone for your attention on this project. Two important things that we want to clarify: 9 | - This is not the official repository of DSSM project. 10 | - We are not authorized to provide any training or test data that belongs to original authors. 11 | 12 | Sorry for any inconveniences caused. 13 | -------------------------------------------------------------------------------- /distributed_dssm/dist/dist-dssm-time-tf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/distributed_dssm/dist/dist-dssm-time-tf.png -------------------------------------------------------------------------------- /distributed_dssm/dist/dist-plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | 6 | num_workers = range(1,8) 7 | time = [769, 400, 370, 300, 285, 260, 242] 8 | 9 | plt.grid() 10 | 11 | barwidth = 0.3 12 | x = np.arange(7) - barwidth / 2 + 1 13 | barlist = plt.bar(x, time, barwidth) 14 | 15 | plt.xticks(num_workers) 16 | plt.title('Training Time Per Epoch') 17 | plt.ylabel("Time (s)") 18 | plt.xlabel("Number of Workers") 19 | plt.xlim(0.5, len(num_workers) + 0.5) 20 | 21 | plt.show() 22 | -------------------------------------------------------------------------------- /distributed_dssm/dist/dist-tf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/distributed_dssm/dist/dist-tf.png -------------------------------------------------------------------------------- /distributed_dssm/tf-datasize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/distributed_dssm/tf-datasize.png -------------------------------------------------------------------------------- /estimators/cifar10_estimator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/estimators/cifar10_estimator/__init__.py -------------------------------------------------------------------------------- /estimators/cifar10_estimator/cmle_config.yaml: -------------------------------------------------------------------------------- 1 | trainingInput: 2 | scaleTier: CUSTOM 3 | masterType: complex_model_m_gpu 4 | workerType: complex_model_m_gpu 5 | parameterServerType: complex_model_m 6 | workerCount: 1 7 | -------------------------------------------------------------------------------- /google-seq2seq/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = 3 | seq2seq 4 | 5 | include = 6 | seq2seq/* 7 | seq2seq/data/* 8 | seq2seq/decoders/* 9 | seq2seq/encoders/* 10 | seq2seq/inference/* 11 | seq2seq/metrics/* 12 | seq2seq/models/* 13 | seq2seq/tasks/* 14 | seq2seq/training/* 15 | 16 | omit = 17 | seq2seq/contrib/* 18 | seq2seq/test/* 19 | seq2seq/scripts/* 20 | 21 | [report] 22 | show_missing=True -------------------------------------------------------------------------------- /google-seq2seq/.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = google 3 | indent_width = 2 4 | column_limit = 80 -------------------------------------------------------------------------------- /google-seq2seq/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution, 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult [GitHub Help] for more 22 | information on using pull requests. 23 | 24 | [GitHub Help]: https://help.github.com/articles/about-pull-requests/ -------------------------------------------------------------------------------- /google-seq2seq/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/google-seq2seq/bin/__init__.py -------------------------------------------------------------------------------- /google-seq2seq/bin/tools/beam_search_viz/tree.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | .node { 19 | cursor: pointer; 20 | } 21 | 22 | .node circle { 23 | fill: #fff; 24 | stroke: steelblue; 25 | stroke-width: 3px; 26 | } 27 | 28 | .node text { 29 | font: 11px sans-serif; 30 | } 31 | 32 | .link { 33 | fill: none; 34 | stroke: #ccc; 35 | stroke-width: 2px; 36 | } 37 | -------------------------------------------------------------------------------- /google-seq2seq/circle.yml: -------------------------------------------------------------------------------- 1 | general: 2 | branches: 3 | ignore: 4 | - gh-pages 5 | 6 | dependencies: 7 | pre: 8 | - sudo apt-get update; sudo apt-get install python-matplotlib python3-matplotlib python-tk python3-tk libtcmalloc-minimal4 9 | override: 10 | - pip install tox tox-pyenv mkdocs 11 | - pyenv local 2.7.10 3.5.1 12 | 13 | machine: 14 | environment: 15 | LD_PRELOAD: /usr/lib/libtcmalloc_minimal.so.4 16 | 17 | test: 18 | pre: 19 | - mkdir -p $HOME/.config/matplotlib 20 | - "echo 'backend : Agg' >> $HOME/.config/matplotlib/matplotlibrc" 21 | override: 22 | - tox 23 | -------------------------------------------------------------------------------- /google-seq2seq/docs/extra.css: -------------------------------------------------------------------------------- 1 | .wy-nav-content { 2 | max-width: 1200px !important; 3 | } -------------------------------------------------------------------------------- /google-seq2seq/docs/help.md: -------------------------------------------------------------------------------- 1 | ## Getting Help 2 | 3 | If you run into problems or find bugs in the code, please file a [Github Issue](https://github.com/google/seq2seq/issues). -------------------------------------------------------------------------------- /google-seq2seq/docs/image_captioning.md: -------------------------------------------------------------------------------- 1 | ## Coming Soon 2 | 3 | This tutorial is coming soon. It is easy to swap out the RNN encoder with a Convolutional Neural Network to perform image captioning. -------------------------------------------------------------------------------- /google-seq2seq/docs/images/nmt_tutorial_bleu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/google-seq2seq/docs/images/nmt_tutorial_bleu.png -------------------------------------------------------------------------------- /google-seq2seq/docs/images/nmt_tutorial_ppl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/google-seq2seq/docs/images/nmt_tutorial_ppl.png -------------------------------------------------------------------------------- /google-seq2seq/docs/license.md: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | The code is available under the [Apache License](https://www.apache.org/licenses/LICENSE-2.0). 4 | 5 | ## Citation 6 | 7 | If you use this code for academic purposes, plase cite it as: 8 | 9 | ``` 10 | @ARTICLE{Britz:2017, 11 | author = {{Britz}, D. and {Goldie}, A. and {Luong}, T. and {Le}, Q.}, 12 | title = "{Massive Exploration of Neural Machine Translation Architectures}", 13 | journal = {ArXiv e-prints}, 14 | archivePrefix = "arXiv", 15 | eprinttype = {arxiv}, 16 | eprint = {1703.03906}, 17 | primaryClass = "cs.CL", 18 | keywords = {Computer Science - Computation and Language}, 19 | year = 2017, 20 | month = mar, 21 | } 22 | ``` 23 | -------------------------------------------------------------------------------- /google-seq2seq/docs/summarization.md: -------------------------------------------------------------------------------- 1 | ## Coming Soon 2 | 3 | Training a summarization model is very similar to [training a Neural Machine Translation](nmt/). Please refer to NMT tutorial for the time being while we are working on a summarization-specific tutorial. -------------------------------------------------------------------------------- /google-seq2seq/example_configs/nmt_conv.yml: -------------------------------------------------------------------------------- 1 | model: AttentionSeq2Seq 2 | model_params: 3 | attention.class: seq2seq.decoders.attention.AttentionLayerBahdanau 4 | attention.params: 5 | num_units: 512 6 | bridge.class: seq2seq.models.bridges.ZeroBridge 7 | embedding.dim: 512 8 | encoder.class: seq2seq.encoders.ConvEncoder 9 | encoder.params: 10 | attention_cnn.units: 512 11 | attention_cnn.kernel_size: 3 12 | attention_cnn.layers: 15 13 | output_cnn.units: 256 14 | output_cnn.kernel_size: 3 15 | output_cnn.layers: 5 16 | position_embeddings.enable: true 17 | position_embeddings.combiner_fn: tensorflow.multiply 18 | position_embeddings.num_positions: 52 19 | decoder.class: seq2seq.decoders.AttentionDecoder 20 | decoder.params: 21 | rnn_cell: 22 | cell_class: LSTMCell 23 | cell_params: 24 | num_units: 512 25 | dropout_input_keep_prob: 0.8 26 | dropout_output_keep_prob: 1.0 27 | num_layers: 4 28 | optimizer.name: Adam 29 | optimizer.learning_rate: 0.0001 30 | source.max_seq_len: 50 31 | source.reverse: false 32 | target.max_seq_len: 50 33 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/nmt_conv_small.yml: -------------------------------------------------------------------------------- 1 | model: AttentionSeq2Seq 2 | model_params: 3 | attention.class: seq2seq.decoders.attention.AttentionLayerBahdanau 4 | attention.params: 5 | num_units: 128 6 | bridge.class: seq2seq.models.bridges.ZeroBridge 7 | embedding.dim: 128 8 | encoder.class: seq2seq.encoders.ConvEncoder 9 | encoder.params: 10 | attention_cnn.units: 128 11 | attention_cnn.kernel_size: 3 12 | attention_cnn.layers: 6 13 | output_cnn.units: 128 14 | output_cnn.kernel_size: 3 15 | output_cnn.layers: 3 16 | position_embeddings.enable: true 17 | position_embeddings.combiner_fn: tensorflow.multiply 18 | position_embeddings.num_positions: 52 19 | decoder.class: seq2seq.decoders.AttentionDecoder 20 | decoder.params: 21 | rnn_cell: 22 | cell_class: GRUCell 23 | cell_params: 24 | num_units: 128 25 | dropout_input_keep_prob: 0.8 26 | dropout_output_keep_prob: 1.0 27 | num_layers: 1 28 | optimizer.name: Adam 29 | optimizer.learning_rate: 0.0001 30 | source.max_seq_len: 50 31 | source.reverse: false 32 | target.max_seq_len: 50 33 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/nmt_large.yml: -------------------------------------------------------------------------------- 1 | model: AttentionSeq2Seq 2 | model_params: 3 | attention.class: seq2seq.decoders.attention.AttentionLayerBahdanau 4 | attention.params: 5 | num_units: 512 6 | bridge.class: seq2seq.models.bridges.ZeroBridge 7 | embedding.dim: 512 8 | encoder.class: seq2seq.encoders.BidirectionalRNNEncoder 9 | encoder.params: 10 | rnn_cell: 11 | cell_class: LSTMCell 12 | cell_params: 13 | num_units: 512 14 | dropout_input_keep_prob: 0.8 15 | dropout_output_keep_prob: 1.0 16 | num_layers: 2 17 | decoder.class: seq2seq.decoders.AttentionDecoder 18 | decoder.params: 19 | rnn_cell: 20 | cell_class: LSTMCell 21 | cell_params: 22 | num_units: 512 23 | dropout_input_keep_prob: 0.8 24 | dropout_output_keep_prob: 1.0 25 | num_layers: 4 26 | optimizer.name: Adam 27 | optimizer.params: 28 | epsilon: 0.0000008 29 | optimizer.learning_rate: 0.0001 30 | source.max_seq_len: 50 31 | source.reverse: false 32 | target.max_seq_len: 50 33 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/nmt_medium.yml: -------------------------------------------------------------------------------- 1 | model: AttentionSeq2Seq 2 | model_params: 3 | attention.class: seq2seq.decoders.attention.AttentionLayerBahdanau 4 | attention.params: 5 | num_units: 256 6 | bridge.class: seq2seq.models.bridges.ZeroBridge 7 | embedding.dim: 256 8 | encoder.class: seq2seq.encoders.BidirectionalRNNEncoder 9 | encoder.params: 10 | rnn_cell: 11 | cell_class: GRUCell 12 | cell_params: 13 | num_units: 256 14 | dropout_input_keep_prob: 0.8 15 | dropout_output_keep_prob: 1.0 16 | num_layers: 1 17 | decoder.class: seq2seq.decoders.AttentionDecoder 18 | decoder.params: 19 | rnn_cell: 20 | cell_class: GRUCell 21 | cell_params: 22 | num_units: 256 23 | dropout_input_keep_prob: 0.8 24 | dropout_output_keep_prob: 1.0 25 | num_layers: 2 26 | optimizer.name: Adam 27 | optimizer.params: 28 | epsilon: 0.0000008 29 | optimizer.learning_rate: 0.0001 30 | source.max_seq_len: 50 31 | source.reverse: false 32 | target.max_seq_len: 50 33 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/nmt_small.yml: -------------------------------------------------------------------------------- 1 | model: AttentionSeq2Seq 2 | model_params: 3 | attention.class: seq2seq.decoders.attention.AttentionLayerDot 4 | attention.params: 5 | num_units: 128 6 | bridge.class: seq2seq.models.bridges.ZeroBridge 7 | embedding.dim: 128 8 | encoder.class: seq2seq.encoders.BidirectionalRNNEncoder 9 | encoder.params: 10 | rnn_cell: 11 | cell_class: GRUCell 12 | cell_params: 13 | num_units: 128 14 | dropout_input_keep_prob: 0.8 15 | dropout_output_keep_prob: 1.0 16 | num_layers: 1 17 | decoder.class: seq2seq.decoders.AttentionDecoder 18 | decoder.params: 19 | rnn_cell: 20 | cell_class: GRUCell 21 | cell_params: 22 | num_units: 128 23 | dropout_input_keep_prob: 0.8 24 | dropout_output_keep_prob: 1.0 25 | num_layers: 1 26 | optimizer.name: Adam 27 | optimizer.params: 28 | epsilon: 0.0000008 29 | optimizer.learning_rate: 0.0001 30 | source.max_seq_len: 50 31 | source.reverse: false 32 | target.max_seq_len: 50 -------------------------------------------------------------------------------- /google-seq2seq/example_configs/text_metrics_bpe.yml: -------------------------------------------------------------------------------- 1 | default_params: &default_params 2 | - separator: " " 3 | - postproc_fn: "seq2seq.data.postproc.strip_bpe" 4 | 5 | metrics: 6 | - class: LogPerplexityMetricSpec 7 | - class: BleuMetricSpec 8 | params: 9 | <<: *default_params 10 | - class: RougeMetricSpec 11 | params: 12 | <<: *default_params 13 | rouge_type: rouge_1/f_score 14 | - class: RougeMetricSpec 15 | params: 16 | <<: *default_params 17 | rouge_type: rouge_1/r_score 18 | - class: RougeMetricSpec 19 | params: 20 | <<: *default_params 21 | rouge_type: rouge_1/p_score 22 | - class: RougeMetricSpec 23 | params: 24 | <<: *default_params 25 | rouge_type: rouge_2/f_score 26 | - class: RougeMetricSpec 27 | params: 28 | <<: *default_params 29 | rouge_type: rouge_2/r_score 30 | - class: RougeMetricSpec 31 | params: 32 | <<: *default_params 33 | rouge_type: rouge_2/p_score 34 | - class: RougeMetricSpec 35 | params: 36 | <<: *default_params 37 | rouge_type: rouge_l/f_score 38 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/text_metrics_sp.yml: -------------------------------------------------------------------------------- 1 | default_params: &default_params 2 | - separator: " " 3 | - postproc_fn: "seq2seq.data.postproc.decode_sentencepiece" 4 | 5 | metrics: 6 | - class: LogPerplexityMetricSpec 7 | - class: BleuMetricSpec 8 | params: 9 | <<: *default_params 10 | - class: RougeMetricSpec 11 | params: 12 | <<: *default_params 13 | rouge_type: rouge_1/f_score 14 | - class: RougeMetricSpec 15 | params: 16 | <<: *default_params 17 | rouge_type: rouge_1/r_score 18 | - class: RougeMetricSpec 19 | params: 20 | <<: *default_params 21 | rouge_type: rouge_1/p_score 22 | - class: RougeMetricSpec 23 | params: 24 | <<: *default_params 25 | rouge_type: rouge_2/f_score 26 | - class: RougeMetricSpec 27 | params: 28 | <<: *default_params 29 | rouge_type: rouge_2/r_score 30 | - class: RougeMetricSpec 31 | params: 32 | <<: *default_params 33 | rouge_type: rouge_2/p_score 34 | - class: RougeMetricSpec 35 | params: 36 | <<: *default_params 37 | rouge_type: rouge_l/f_score 38 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/train_seq2seq.yml: -------------------------------------------------------------------------------- 1 | buckets: 10,20,30,40 2 | hooks: 3 | - class: PrintModelAnalysisHook 4 | - class: MetadataCaptureHook 5 | - class: SyncReplicasOptimizerHook 6 | - class: TrainSampleHook 7 | params: 8 | every_n_steps: 1000 9 | -------------------------------------------------------------------------------- /google-seq2seq/example_configs/train_seq2seq_delay_start.yml: -------------------------------------------------------------------------------- 1 | buckets: 10,20,30,40 2 | hooks: 3 | - class: PrintModelAnalysisHook 4 | - class: MetadataCaptureHook 5 | - class: DelayStartHook 6 | - class: TrainSampleHook 7 | params: 8 | every_n_steps: 1000 9 | -------------------------------------------------------------------------------- /google-seq2seq/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: seq2seq 2 | theme: readthedocs 3 | extra_css: [extra.css] 4 | markdown_extensions: [fenced_code] 5 | pages: 6 | - Overview: index.md 7 | - Getting Started: getting_started.md 8 | - Concepts: concepts.md 9 | - "Tutorial: Neural Machine Translation": nmt.md 10 | - "Tutorial: Summarization": summarization.md 11 | - "Tutorial: Image Captioning": image_captioning.md 12 | - Data: data.md 13 | - Training: training.md 14 | - Inference: inference.md 15 | - Tools: tools.md 16 | - Results: results.md 17 | - Getting Help: help.md 18 | - Contributing: contributing.md 19 | - License: license.md 20 | - "Reference: Models": models.md 21 | - "Reference: Encoders": encoders.md 22 | - "Reference: Decoders": decoders.md 23 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | seq2seq library base module 16 | """ 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | from seq2seq.graph_module import GraphModule 23 | 24 | from seq2seq import contrib 25 | from seq2seq import data 26 | from seq2seq import decoders 27 | from seq2seq import encoders 28 | from seq2seq import global_vars 29 | from seq2seq import graph_utils 30 | from seq2seq import inference 31 | from seq2seq import losses 32 | from seq2seq import metrics 33 | from seq2seq import models 34 | from seq2seq import test 35 | from seq2seq import training 36 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/contrib/seq2seq/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Collection of input-related utlities. 15 | """ 16 | 17 | from seq2seq.data import input_pipeline 18 | from seq2seq.data import parallel_data_provider 19 | from seq2seq.data import postproc 20 | from seq2seq.data import split_tokens_decoder 21 | from seq2seq.data import vocab 22 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/decoders/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Collection of decoders and decoder-related functions. 15 | """ 16 | 17 | from seq2seq.decoders.rnn_decoder import * 18 | from seq2seq.decoders.attention import * 19 | from seq2seq.decoders.basic_decoder import * 20 | from seq2seq.decoders.attention_decoder import * 21 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Collection of encoders""" 15 | 16 | import seq2seq.encoders.encoder 17 | import seq2seq.encoders.rnn_encoder 18 | 19 | from seq2seq.encoders.rnn_encoder import * 20 | from seq2seq.encoders.image_encoder import * 21 | from seq2seq.encoders.pooling_encoder import PoolingEncoder 22 | from seq2seq.encoders.conv_encoder import ConvEncoder 23 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/global_vars.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Collection of global variables. 16 | """ 17 | 18 | SYNC_REPLICAS_OPTIMIZER = None 19 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/inference/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Modules related to running model inference. 15 | """ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from seq2seq.inference.inference import * 22 | import seq2seq.inference.beam_search 23 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ Collection of metric-related functions 15 | """ 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """This module contains various Encoder-Decoder models 15 | """ 16 | 17 | from seq2seq.models.basic_seq2seq import BasicSeq2Seq 18 | from seq2seq.models.attention_seq2seq import AttentionSeq2Seq 19 | from seq2seq.models.image2seq import Image2Seq 20 | 21 | import seq2seq.models.bridges 22 | import seq2seq.models.model_base 23 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Collection of task types. 16 | """ 17 | 18 | from seq2seq.tasks.inference_task import InferenceTask 19 | from seq2seq.tasks.decode_text import DecodeText 20 | from seq2seq.tasks.dump_attention import DumpAttention 21 | from seq2seq.tasks.dump_beams import DumpBeams 22 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests and testing utilities 15 | """ 16 | 17 | from seq2seq.test import utils 18 | -------------------------------------------------------------------------------- /google-seq2seq/seq2seq/training/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Operatations and wrappers to help with model training. 15 | """ 16 | 17 | from seq2seq.training import hooks 18 | from seq2seq.training import utils 19 | -------------------------------------------------------------------------------- /google-seq2seq/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Python package setup file. 17 | """ 18 | 19 | from setuptools import setup 20 | 21 | setup( 22 | name="seq2seq", 23 | version="0.1", 24 | install_requires=[ 25 | "numpy", 26 | "matplotlib", 27 | "pyyaml", 28 | "pyrouge" 29 | ], 30 | extras_require={'tensorflow': ['tensorflow'], 31 | 'tensorflow with gpu': ['tensorflow-gpu']}, 32 | ) 33 | -------------------------------------------------------------------------------- /google-seq2seq/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py35,pylint 3 | 4 | [testenv] 5 | passenv=CIRCLE_ARTIFACTS LD_PRELOAD 6 | deps= 7 | nose 8 | coverage 9 | tensorflow 10 | commands= 11 | coverage run {envbindir}/nosetests 12 | coverage report -m 13 | coverage html -d {env:CIRCLE_ARTIFACTS:/tmp/tox}/{envname}_coverage 14 | 15 | [testenv:pylint] 16 | basepython=python3.5 17 | deps= 18 | tensorflow 19 | pylint 20 | commands=pylint -E seq2seq 21 | 22 | [testenv:py35] 23 | deps= 24 | nose 25 | coverage 26 | tensorflow 27 | 28 | [testenv:py27] 29 | deps= 30 | nose 31 | coverage 32 | tensorflow 33 | -------------------------------------------------------------------------------- /kaggle-avazu-master/AvazuModelDocumentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/kaggle-avazu-master/AvazuModelDocumentation.pdf -------------------------------------------------------------------------------- /kaggle-avazu-master/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2015 Zhonghua Zhang 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | this file except in compliance with the License. You may obtain a copy of the 5 | License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by 6 | applicable law or agreed to in writing, software distributed under the License 7 | is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | KIND, either express or implied. See the License for the specific language 9 | governing permissions and limitations under the License. 10 | -------------------------------------------------------------------------------- /kaggle-avazu-master/READ.me: -------------------------------------------------------------------------------- 1 | To reproduce the result: 2 | 3 | 1. Clone the github repo, go to the folder 4 | 2. Set paths in the utils.py file 5 | 3. sh ./_0_run_me.sh 6 | 7 | 8 | The shell script will run the model 3 times 9 | 10 | 1. a small sample run using day 30 as validation -- should take about 1-2 hours and generate .393-394 logloss 11 | 2. a small sample run using day 31 as test -- should get LB score about .391-.392 12 | 3. a full run using day 31 as test -- should get LB score that ranks 2nd. 13 | 14 | Please note that the full run will take about 2 days, and require 130GB of temporary storage space. So highly recommended run it over the weekend, after step 1 and 2 are successful. 15 | 16 | This process CAN be made much more efficient with a few hours work by a REAL software engineer. 17 | 18 | -------------------------------------------------------------------------------- /my_test_serving/remote_copy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | echo "scp to gpu03, password=gIWUNipp23" 5 | scp -r *.{py,sh,txt,json} root@gpu03:/data/zhaocl/test_tf_serving 6 | -------------------------------------------------------------------------------- /my_test_serving/simple_math1_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import tensorflow as tf 4 | 5 | x = tf.placeholder(tf.float32, shape=(None)) 6 | y = tf.placeholder(tf.float32, shape=(None)) 7 | 8 | three = tf.Variable(3, dtype=tf.float32) 9 | z = tf.scalar_mul(three, x) + y 10 | 11 | model_version = 1 12 | path = os.path.join("three_x_plus_y", str(model_version)) 13 | builder = tf.saved_model.builder.SavedModelBuilder(path) 14 | 15 | with tf.Session() as sess: 16 | sess.run(tf.global_variables_initializer()) 17 | 18 | builder.add_meta_graph_and_variables( 19 | sess, 20 | [tf.saved_model.tag_constants.SERVING], 21 | signature_def_map={ 22 | "magic_model": tf.saved_model.signature_def_utils.predict_signature_def( 23 | inputs={"egg": x, "bacon": y}, 24 | outputs={"spam": z}) 25 | }) 26 | builder.save() 27 | -------------------------------------------------------------------------------- /nlp_estimator_tutorial/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Julian Eisenschlos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /nlp_estimator_tutorial/README.md: -------------------------------------------------------------------------------- 1 | # nlp_estimator_tutorial 2 | Educational material on using the TensorFlow Estimator framework for text classification 3 | 4 | Try it live in Colab [here](https://colab.research.google.com/github/eisenjulian/nlp_estimator_tutorial/blob/master/nlp_estimators.ipynb) 5 | -------------------------------------------------------------------------------- /nlp_estimator_tutorial/conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/conv.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/embeddings.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/embeddings.gif -------------------------------------------------------------------------------- /nlp_estimator_tutorial/embeddings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/embeddings.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/loss.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/mermaid/cnn.mmd: -------------------------------------------------------------------------------- 1 | graph LR 2 | id1(Embedding Layer) --> id2(Dropout) 3 | id2 --> id3(Convolution1D) 4 | id3 --> id4(GlobalMaxPooling1D) 5 | id4 --> id5(Hidden Dense Layer) 6 | id5 --> id6(Dropout) 7 | id6 --> id7(Output Layer) 8 | -------------------------------------------------------------------------------- /nlp_estimator_tutorial/mermaid/cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/mermaid/cnn.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/mermaid/lstm.mmd: -------------------------------------------------------------------------------- 1 | graph LR 2 | id1(Embedding Layer) --> id2(LSTM Cell) 3 | id2 -->|Recursion| id2 4 | id2 --> id3(Output Layer) -------------------------------------------------------------------------------- /nlp_estimator_tutorial/mermaid/lstm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/mermaid/lstm.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/pr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/pr.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/pr_curves.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/pr_curves.png -------------------------------------------------------------------------------- /nlp_estimator_tutorial/token_weights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nlp_estimator_tutorial/token_weights.png -------------------------------------------------------------------------------- /nmt/nmt/.gitignore: -------------------------------------------------------------------------------- 1 | bazel-bin 2 | bazel-genfiles 3 | bazel-out 4 | bazel-testlogs 5 | -------------------------------------------------------------------------------- /nmt/nmt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/__init__.py -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/attention_equation_0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/attention_equation_0.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/attention_equation_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/attention_equation_1.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/attention_mechanism.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/attention_mechanism.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/attention_vis.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/attention_vis.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/encdec.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/encdec.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/greedy_dec.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/greedy_dec.jpg -------------------------------------------------------------------------------- /nmt/nmt/g3doc/img/seq2seq.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/g3doc/img/seq2seq.jpg -------------------------------------------------------------------------------- /nmt/nmt/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/scripts/__init__.py -------------------------------------------------------------------------------- /nmt/nmt/standard_hparams/iwslt15.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention": "scaled_luong", 3 | "attention_architecture": "standard", 4 | "batch_size": 128, 5 | "bpe_delimiter": null, 6 | "colocate_gradients_with_ops": true, 7 | "decay_factor": 0.5, 8 | "decay_steps": 1000, 9 | "dropout": 0.2, 10 | "encoder_type": "bi", 11 | "eos": "", 12 | "forget_bias": 1.0, 13 | "infer_batch_size": 32, 14 | "init_weight": 0.1, 15 | "learning_rate": 1.0, 16 | "max_gradient_norm": 5.0, 17 | "metrics": ["bleu"], 18 | "num_buckets": 5, 19 | "num_layers": 2, 20 | "num_train_steps": 12000, 21 | "num_units": 512, 22 | "optimizer": "sgd", 23 | "residual": false, 24 | "share_vocab": false, 25 | "sos": "", 26 | "source_reverse": false, 27 | "src_max_len": 50, 28 | "src_max_len_infer": null, 29 | "start_decay_step": 8000, 30 | "steps_per_external_eval": null, 31 | "steps_per_stats": 100, 32 | "tgt_max_len": 50, 33 | "tgt_max_len_infer": null, 34 | "time_major": true, 35 | "unit_type": "lstm", 36 | "beam_width": 10 37 | } 38 | -------------------------------------------------------------------------------- /nmt/nmt/standard_hparams/wmt16.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention": "normed_bahdanau", 3 | "attention_architecture": "standard", 4 | "batch_size": 128, 5 | "bpe_delimiter": "@@", 6 | "colocate_gradients_with_ops": true, 7 | "decay_factor": 0.5, 8 | "decay_steps": 17000, 9 | "dropout": 0.2, 10 | "encoder_type": "bi", 11 | "eos": "", 12 | "forget_bias": 1.0, 13 | "infer_batch_size": 32, 14 | "init_weight": 0.1, 15 | "learning_rate": 1.0, 16 | "max_gradient_norm": 5.0, 17 | "metrics": ["bleu"], 18 | "num_buckets": 5, 19 | "num_layers": 4, 20 | "num_train_steps": 340000, 21 | "num_units": 1024, 22 | "optimizer": "sgd", 23 | "residual": false, 24 | "share_vocab": false, 25 | "sos": "", 26 | "source_reverse": false, 27 | "src_max_len": 50, 28 | "src_max_len_infer": null, 29 | "start_decay_step": 170000, 30 | "steps_per_external_eval": null, 31 | "steps_per_stats": 100, 32 | "tgt_max_len": 50, 33 | "tgt_max_len_infer": null, 34 | "time_major": true, 35 | "unit_type": "lstm", 36 | "beam_width": 10 37 | } 38 | -------------------------------------------------------------------------------- /nmt/nmt/standard_hparams/wmt16_gnmt_4_layer.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention": "normed_bahdanau", 3 | "attention_architecture": "gnmt_v2", 4 | "batch_size": 128, 5 | "bpe_delimiter": "@@", 6 | "colocate_gradients_with_ops": true, 7 | "decay_factor": 0.5, 8 | "decay_steps": 17000, 9 | "dropout": 0.2, 10 | "encoder_type": "gnmt", 11 | "eos": "", 12 | "forget_bias": 1.0, 13 | "infer_batch_size": 32, 14 | "init_weight": 0.1, 15 | "learning_rate": 1.0, 16 | "max_gradient_norm": 5.0, 17 | "metrics": ["bleu"], 18 | "num_buckets": 5, 19 | "num_layers": 4, 20 | "num_train_steps": 340000, 21 | "num_units": 1024, 22 | "optimizer": "sgd", 23 | "residual": true, 24 | "share_vocab": false, 25 | "sos": "", 26 | "source_reverse": false, 27 | "src_max_len": 50, 28 | "src_max_len_infer": null, 29 | "start_decay_step": 170000, 30 | "steps_per_external_eval": null, 31 | "steps_per_stats": 100, 32 | "tgt_max_len": 50, 33 | "tgt_max_len_infer": null, 34 | "time_major": true, 35 | "unit_type": "lstm", 36 | "beam_width": 10, 37 | "length_penalty_weight": 1.0 38 | } 39 | -------------------------------------------------------------------------------- /nmt/nmt/standard_hparams/wmt16_gnmt_8_layer.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention": "normed_bahdanau", 3 | "attention_architecture": "gnmt_v2", 4 | "batch_size": 128, 5 | "bpe_delimiter": "@@", 6 | "colocate_gradients_with_ops": true, 7 | "decay_factor": 0.5, 8 | "decay_steps": 17000, 9 | "dropout": 0.2, 10 | "encoder_type": "gnmt", 11 | "eos": "", 12 | "forget_bias": 1.0, 13 | "infer_batch_size": 32, 14 | "init_weight": 0.1, 15 | "learning_rate": 1.0, 16 | "max_gradient_norm": 5.0, 17 | "metrics": ["bleu"], 18 | "num_buckets": 5, 19 | "num_layers": 8, 20 | "num_train_steps": 340000, 21 | "num_units": 1024, 22 | "optimizer": "sgd", 23 | "residual": true, 24 | "share_vocab": false, 25 | "sos": "", 26 | "source_reverse": false, 27 | "src_max_len": 50, 28 | "src_max_len_infer": null, 29 | "start_decay_step": 170000, 30 | "steps_per_external_eval": null, 31 | "steps_per_stats": 100, 32 | "tgt_max_len": 50, 33 | "tgt_max_len_infer": null, 34 | "time_major": true, 35 | "unit_type": "lstm", 36 | "beam_width": 10, 37 | "length_penalty_weight": 1.0 38 | } 39 | -------------------------------------------------------------------------------- /nmt/nmt/testdata/label_ref: -------------------------------------------------------------------------------- 1 | positive 2 | positive 3 | positive 4 | negative 5 | negative -------------------------------------------------------------------------------- /nmt/nmt/testdata/pred_output: -------------------------------------------------------------------------------- 1 | positive 2 | positive 3 | negative 4 | negative 5 | positive -------------------------------------------------------------------------------- /nmt/nmt/testdata/test_embed.txt: -------------------------------------------------------------------------------- 1 | some_word 1.0 2.0 3.0 4.0 2 | some_other_word 4.0 3.0 2.0 1.0 3 | -------------------------------------------------------------------------------- /nmt/nmt/testdata/test_embed_with_header.txt: -------------------------------------------------------------------------------- 1 | 2 4 2 | some_word 1.0 2.0 3.0 4.0 3 | some_other_word 4.0 3.0 2.0 1.0 4 | -------------------------------------------------------------------------------- /nmt/nmt/testdata/test_infer_file: -------------------------------------------------------------------------------- 1 | A Republic@@ an strategy to counter the re-@@ election of Obama 2 | Republic@@ an leaders justified their policy by the need to combat electoral fraud . 3 | However , the Brenn@@ an Centre considers this a my@@ th , stating that electoral fraud is rar@@ er in the United States than the number of people killed by ligh@@ tn@@ ing . 4 | Indeed , Republic@@ an lawyers identified only 300 cases of electoral fraud in the United States in a decade . 5 | One thing is certain : these new provisions will have a negative impact on vot@@ er tur@@ n-@@ out . -------------------------------------------------------------------------------- /nmt/nmt/testdata/test_infer_vocab.src: -------------------------------------------------------------------------------- 1 | unk 2 | eos 3 | sos 4 | test1 5 | test2 6 | -------------------------------------------------------------------------------- /nmt/nmt/testdata/test_infer_vocab.tgt: -------------------------------------------------------------------------------- 1 | unk 2 | eos 3 | test1 4 | test2 5 | test3 6 | test4 7 | -------------------------------------------------------------------------------- /nmt/nmt/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/nmt/nmt/utils/__init__.py -------------------------------------------------------------------------------- /tensorflow-DeepFM/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Chenglong Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tensorflow-DeepFM/example/README.md: -------------------------------------------------------------------------------- 1 | 2 | An example usage of DeepFM/FM/DNN models for [Porto Seguro's Safe Driver Prediction competition on Kaggle](https://www.kaggle.com/c/porto-seguro-safe-driver-prediction). 3 | -------------------------------------------------------------------------------- /tensorflow-DeepFM/example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow-DeepFM/example/__init__.py -------------------------------------------------------------------------------- /tensorflow-DeepFM/example/data/README.md: -------------------------------------------------------------------------------- 1 | 2 | Please download the data from the [competition website](https://www.kaggle.com/c/porto-seguro-safe-driver-prediction) and put them here. 3 | -------------------------------------------------------------------------------- /tensorflow-DeepFM/example/metrics.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | 4 | def gini(actual, pred): 5 | assert (len(actual) == len(pred)) 6 | all = np.asarray(np.c_[actual, pred, np.arange(len(actual))], dtype=np.float) 7 | all = all[np.lexsort((all[:, 2], -1 * all[:, 1]))] 8 | totalLosses = all[:, 0].sum() 9 | giniSum = all[:, 0].cumsum().sum() / totalLosses 10 | 11 | giniSum -= (len(actual) + 1) / 2. 12 | return giniSum / len(actual) 13 | 14 | def gini_norm(actual, pred): 15 | return gini(actual, pred) / gini(actual, actual) 16 | -------------------------------------------------------------------------------- /tensorflow-DeepFM/example/output/README.md: -------------------------------------------------------------------------------- 1 | 2 | Submissions are saved here. 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/README.md: -------------------------------------------------------------------------------- 1 | # Cpp Predict Client 2 | 3 | ## Introduction 4 | 5 | TensorFlow serving is the gRPC service for general TensorFlow models. We can implement the C++ gRPC client to predict. 6 | 7 | Now you need to use `bazel` and just refer to to [inception_client.cc](https://github.com/tensorflow/serving/pull/300). 8 | 9 | ## Usage 10 | 11 | Add the build rule in `tensorflow_serving/example/BUILD` and copy [sparse_predict_client.cc](./sparse_predict_client.cc) in example direcotry. 12 | 13 | ``` 14 | cc_binary( 15 | name = "sparse_predict_client", 16 | srcs = [ 17 | "sparse_predict_client.cc", 18 | ], 19 | deps = [ 20 | "//tensorflow_serving/apis:prediction_service_proto", 21 | ], 22 | ) 23 | ``` 24 | 25 | Compile the gRPC client. 26 | 27 | ``` 28 | bazel build //tensorflow_serving/example:sparse_predict_client 29 | ``` 30 | 31 | Run the predict client. 32 | 33 | ``` 34 | bazel-bin/tensorflow_serving/example/sparse_predict_client 35 | ``` 36 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/generate_proto_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | protoc -I ./ --cpp_out=.. **/*.proto 7 | protoc -I ./ --grpc_out=.. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` **/*.proto 8 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "predict.proto"; 7 | 8 | // PredictionService provides access to machine-learned models loaded by 9 | // model_servers. 10 | service PredictionService { 11 | // Predict -- provides access to loaded TensorFlow model. 12 | rpc Predict(PredictRequest) returns (PredictResponse); 13 | } 14 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/contrib/ios_examples/.gitignore: -------------------------------------------------------------------------------- 1 | project.xcworkspace 2 | xcuserdata 3 | imagenet_comp_graph_label_strings.txt 4 | tensorflow_inception_graph.pb 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/contrib/makefile/.gitignore: -------------------------------------------------------------------------------- 1 | gen/ 2 | downloads/ 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/contrib/pi_examples/.gitignore: -------------------------------------------------------------------------------- 1 | tensorflow_inception_graph.pb 2 | imagenet_comp_graph_label_strings.txt 3 | tensorflow_inception_stripped.pb 4 | */gen/ 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/allocation_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "AllocationDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message AllocationDescription { 10 | // Total number of bytes requested 11 | int64 requested_bytes = 1; 12 | 13 | // Total number of bytes allocated if known 14 | int64 allocated_bytes = 2; 15 | 16 | // Name of the allocator used 17 | string allocator_name = 3; 18 | 19 | // Identifier of the allocated buffer if known 20 | int64 allocation_id = 4; 21 | 22 | // Set if this tensor only has one remaining reference 23 | bool has_single_reference = 5; 24 | 25 | // Address of the allocation. 26 | uint64 ptr = 6; 27 | }; 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/device_attributes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "DeviceAttributesProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message DeviceLocality { 10 | // Optional bus locality of device. Default value of 0 means 11 | // no specific locality. Specific localities are indexed from 1. 12 | int32 bus_id = 1; 13 | }; 14 | 15 | message DeviceAttributes { 16 | // Fully specified name of the device within a cluster. 17 | string name = 1; 18 | 19 | // String representation of device_type. 20 | string device_type = 2; 21 | 22 | // Memory capacity of device in bytes. 23 | int64 memory_limit = 4; 24 | 25 | // Platform-specific data about device that may be useful 26 | // for supporting efficient data transfers. 27 | DeviceLocality locality = 5; 28 | 29 | // A device is assigned a global unique number each time it is 30 | // initialized. "incarnation" should never be 0. 31 | fixed64 incarnation = 6; 32 | 33 | // String representation of the physical device that this device maps to. 34 | string physical_device_desc = 7; 35 | } 36 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/kernel_def.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "KernelDefProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/attr_value.proto"; 10 | 11 | message KernelDef { 12 | // Must match the name of an Op. 13 | string op = 1; 14 | 15 | // Type of device this kernel runs on. 16 | string device_type = 2; 17 | 18 | message AttrConstraint { 19 | // Name of an attr from the Op. 20 | string name = 1; 21 | 22 | // A list of values that this kernel supports for this attr. 23 | // Like OpDef.AttrDef.allowed_values, except for kernels instead of Ops. 24 | AttrValue allowed_values = 2; 25 | } 26 | repeated AttrConstraint constraint = 3; 27 | 28 | // Names of the Op's input_/output_args that reside in host memory 29 | // instead of device memory. 30 | repeated string host_memory_arg = 4; 31 | 32 | // This allows experimental kernels to be registered for an op that 33 | // won't be used unless the user specifies a "_kernel" attr with 34 | // value matching this. 35 | string label = 5; 36 | } 37 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ResourceHandleProto"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 10 | // not valid across executions, but can be serialized back and forth from within 11 | // a single run. 12 | message ResourceHandle { 13 | // Unique name for the device containing the resource. 14 | string device = 1; 15 | 16 | // Container in which this resource is placed. 17 | string container = 2; 18 | 19 | // Unique name of this resource. 20 | string name = 3; 21 | 22 | // Hash code for the type of the resource. Is only valid in the same device 23 | // and in the same execution. 24 | uint64 hash_code = 4; 25 | 26 | // For debug-only, the name of the type pointed to by this handle, if 27 | // available. 28 | string maybe_type_name = 5; 29 | }; 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/tensor_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/types.proto"; 10 | import "tensorflow/core/framework/tensor_shape.proto"; 11 | import "tensorflow/core/framework/allocation_description.proto"; 12 | 13 | message TensorDescription { 14 | // Data type of tensor elements 15 | DataType dtype = 1; 16 | 17 | // Shape of the tensor. 18 | TensorShapeProto shape = 2; 19 | 20 | // Information about the size and allocator used for the data 21 | AllocationDescription allocation_description = 4; 22 | }; 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/variable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VariableProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a Variable. 10 | message VariableDef { 11 | // Name of the variable tensor. 12 | string variable_name = 1; 13 | 14 | // Name of the initializer op. 15 | string initializer_name = 2; 16 | 17 | // Name of the snapshot tensor. 18 | string snapshot_name = 3; 19 | 20 | // Support for saving variables as slices of a larger variable. 21 | SaveSliceInfoDef save_slice_info_def = 4; 22 | } 23 | 24 | message SaveSliceInfoDef { 25 | // Name of the full variable of which this is a slice. 26 | string full_name = 1; 27 | // Shape of the full variable. 28 | repeated int32 full_shape = 2; 29 | // Offset of this variable into the full variable. 30 | repeated int32 var_offset = 3; 31 | // Shape of this variable. 32 | repeated int32 var_shape = 4; 33 | } 34 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/framework/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VersionsProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Version information for a piece of serialized data 10 | // 11 | // There are different types of versions for each type of data 12 | // (GraphDef, etc.), but they all have the same common shape 13 | // described here. 14 | // 15 | // Each consumer has "consumer" and "min_producer" versions (specified 16 | // elsewhere). A consumer is allowed to consume this data if 17 | // 18 | // producer >= min_producer 19 | // consumer >= min_consumer 20 | // consumer not in bad_consumers 21 | // 22 | message VersionDef { 23 | // The version of the code that produced this data. 24 | int32 producer = 1; 25 | 26 | // Any consumer below this version is not allowed to consume this data. 27 | int32 min_consumer = 2; 28 | 29 | // Specific consumer versions which are disallowed (e.g. due to bugs). 30 | repeated int32 bad_consumers = 3; 31 | }; 32 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/kernels/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ReaderBaseProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.kernels"; 8 | 9 | // For serializing and restoring the state of ReaderBase, see 10 | // reader_base.h for details. 11 | message ReaderBaseState { 12 | int64 work_started = 1; 13 | int64 work_finished = 2; 14 | int64 num_records_produced = 3; 15 | bytes current_work = 4; 16 | }; 17 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/protobuf/named_tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "NamedTensorProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/tensor.proto"; 10 | 11 | // A pair of tensor name and tensor values. 12 | message NamedTensorProto { 13 | // Name of the tensor. 14 | string name = 1; 15 | 16 | // The client can populate a TensorProto using a tensorflow::Tensor`, or 17 | // directly using the protobuf field accessors. 18 | // 19 | // The client specifies whether the returned tensor values should be 20 | // filled tensor fields (float_val, int_val, etc.) or encoded in a 21 | // compact form in tensor.tensor_content. 22 | TensorProto tensor = 2; 23 | } 24 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/protobuf/queue_runner.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "QueueRunnerProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/lib/core/error_codes.proto"; 10 | 11 | // Protocol buffer representing a QueueRunner. 12 | message QueueRunnerDef { 13 | // Queue name. 14 | string queue_name = 1; 15 | 16 | // A list of enqueue operations. 17 | repeated string enqueue_op_name = 2; 18 | 19 | // The operation to run to close the queue. 20 | string close_op_name = 3; 21 | 22 | // The operation to run to cancel the queue. 23 | string cancel_op_name = 4; 24 | 25 | // A list of exception types considered to signal a safely closed queue 26 | // if raised during enqueue operations. 27 | repeated error.Code queue_closed_exception_types = 5; 28 | } 29 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/protobuf/saved_model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "SavedModelProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/protobuf/meta_graph.proto"; 10 | 11 | // SavedModel is the high level serialization format for TensorFlow Models. 12 | // See [todo: doc links, similar to session_bundle] for more information. 13 | message SavedModel { 14 | // The schema version of the SavedModel instance. Used for versioning when 15 | // making future changes to the specification/implementation. Initial value 16 | // at release will be 1. 17 | int64 saved_model_schema_version = 1; 18 | 19 | // One or more MetaGraphs. 20 | repeated MetaGraphDef meta_graphs = 2; 21 | } 22 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/core/util/memmapped_file_system.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package tensorflow; 18 | option cc_enable_arenas = true; 19 | 20 | // A message that describes one region of memmapped file. 21 | message MemmappedFileSystemDirectoryElement { 22 | uint64 offset = 1; 23 | string name = 2; 24 | } 25 | 26 | // A directory of regions in a memmapped file. 27 | message MemmappedFileSystemDirectory { 28 | repeated MemmappedFileSystemDirectoryElement element = 1; 29 | } 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/examples/udacity/.gitignore: -------------------------------------------------------------------------------- 1 | notMNIST_large* 2 | notMNIST_small* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/go/genop/.gitignore: -------------------------------------------------------------------------------- 1 | # .pb.go files generated by generate.sh 2 | internal/proto/* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/go/op/.gitignore: -------------------------------------------------------------------------------- 1 | wrappers.go 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/python/framework/cpp_shape_inference.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow/core/framework/types.proto"; 7 | import "tensorflow/core/framework/tensor_shape.proto"; 8 | 9 | message CppShapeInferenceResult { 10 | TensorShapeProto shape = 1; 11 | TensorShapeProto handle_shape = 2; 12 | DataType handle_dtype = 3; 13 | } -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/python/training/checkpoint_state.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | // Protocol buffer representing the checkpoint state. 7 | // 8 | // TODO(touts): Add other attributes as needed. 9 | message CheckpointState { 10 | // Path to the most-recent model checkpoint. 11 | string model_checkpoint_path = 1; 12 | 13 | // Paths to all not-yet-deleted model checkpoints, sorted from oldest to 14 | // newest. 15 | // Note that the value of model_checkpoint_path should be the last item in 16 | // this list. 17 | repeated string all_model_checkpoint_paths = 2; 18 | } 19 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/python/util/protobuf/compare_test.proto: -------------------------------------------------------------------------------- 1 | // Test messages used in compare_test.py. 2 | syntax = "proto2"; 3 | 4 | package compare_test; 5 | option cc_enable_arenas = true; 6 | 7 | enum Enum { 8 | A = 0; 9 | B = 1; 10 | C = 2; 11 | } 12 | 13 | message Small { 14 | repeated string strings = 1; 15 | }; 16 | 17 | message Medium { 18 | repeated int32 int32s = 1; 19 | repeated Small smalls = 2; 20 | repeated group GroupA = 3 { 21 | repeated group GroupB = 4 { 22 | required string strings = 5; 23 | } 24 | } 25 | repeated float floats = 6; 26 | }; 27 | 28 | message Large { 29 | optional string string_ = 1; 30 | optional int64 int64_ = 2; 31 | optional float float_ = 3; 32 | optional bool bool_ = 4; 33 | optional Enum enum_ = 5; 34 | repeated int64 int64s = 6; 35 | optional Medium medium = 7; 36 | optional Small small = 8; 37 | optional double double_ = 9; 38 | optional WithMap with_map = 10; 39 | }; 40 | 41 | message Labeled { 42 | required int32 required = 1; 43 | optional int32 optional = 2; 44 | } 45 | 46 | message WithMap { 47 | map value_message = 1; 48 | map value_string = 2; 49 | } 50 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tensorboard/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory" : "components" 3 | } -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tensorboard/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | typings/* 3 | build/* 4 | dist/tf-tensorboard-demo.html 5 | 6 | # Since bower components are stored in the same directory as 7 | # tensorboard components, we ignore everything under components 8 | # except our own components which start with tf-. 9 | components/* 10 | # This rule should always be in sync with TF_COMPONENTS_TYPESCRIPT_GLOB 11 | # in gulpfile.js 12 | !components/tf-* 13 | !components/tf_* 14 | !components/vz-* 15 | !components/vz_* 16 | !components/index.html 17 | !components/BUILD 18 | # Ignore the sample graph files since they are too large to 19 | # be in the repo. 20 | components/tf-graph/demo/tf_model_zoo/* 21 | 22 | # All standalone code for TensorBoard components should be written in 23 | # typescript, and the compiled javascript code should be ignored. 24 | components/tf-*/**/*.js 25 | components/tf_*/**/*.js 26 | components/vz-*/**/*.js 27 | components/vz_*/**/*.js 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tools/ci_build/install/.bazelrc: -------------------------------------------------------------------------------- 1 | # Running bazel inside a `docker build` command causes trouble, cf: 2 | # https://github.com/bazelbuild/bazel/issues/134 3 | # The easiest solution is to set up a bazelrc file forcing --batch. 4 | startup --batch 5 | 6 | # Similarly, we need to workaround sandboxing issues: 7 | # https://github.com/bazelbuild/bazel/issues/418 8 | build --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone 9 | test --spawn_strategy=standalone 10 | 11 | # Force bazel output to use colors (good for jenkins) and print useful errors. 12 | common --color=yes 13 | 14 | # Configure tests - increase timeout, print errors and timeout warnings 15 | test --verbose_failures --test_output=errors --test_verbose_timeout_warnings 16 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tools/git/.gitignore: -------------------------------------------------------------------------------- 1 | gen 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tools/swig/.gitignore: -------------------------------------------------------------------------------- 1 | swig_path 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tools/tfprof/tfprof_log.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | message OpLogEntry { 6 | // op name. 7 | optional string name = 1; 8 | // float_ops is filled by tfprof Python API when called. It requires the 9 | // op has RegisterStatistics defined. Currently, Conv2D, MatMul, etc, are 10 | // implemented. 11 | optional int64 float_ops = 2; 12 | // User can define extra op type information for an op. This allows the user 13 | // to select a group of ops precisely using op_type as a key. 14 | repeated string types = 3; 15 | } 16 | 17 | message OpLog { 18 | repeated OpLogEntry log_entries = 1; 19 | } -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_client/generate_proto_files/tensorflow/tools/tfprof/tfprof_options.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | // Refers to tfprof_options.h/cc for documentation. 6 | // Only used to pass tfprof options from Python to C++. 7 | message OptionsProto { 8 | optional int64 max_depth = 1; 9 | optional int64 min_bytes = 2; 10 | optional int64 min_micros = 3; 11 | optional int64 min_params = 4; 12 | optional int64 min_float_ops = 5; 13 | repeated string device_regexes = 6; 14 | optional string order_by = 7; 15 | repeated string account_type_regexes = 8; 16 | repeated string start_name_regexes = 9; 17 | repeated string trim_name_regexes = 10; 18 | repeated string show_name_regexes = 11; 19 | repeated string hide_name_regexes = 12; 20 | optional bool account_displayed_op_only = 13; 21 | repeated string select = 14; 22 | optional bool viz = 15; 23 | optional string dump_to_file = 16; 24 | } -------------------------------------------------------------------------------- /tensorflow_template_application/cpp_predict_server/README.md: -------------------------------------------------------------------------------- 1 | # Cpp Predict Server 2 | 3 | ## Introduction 4 | 5 | TensorFlow serving is the gRPC server for general TensorFlow models. 6 | 7 | ## Installation 8 | 9 | Follow the official documents to build with `bazel build //tensorflow_serving/model_servers:tensorflow_model_server`. 10 | 11 | Or use the binary [tensorflow_model_server](./tensorflow_model_server/) with TensorFlow 0.11.0 in Linux. 12 | 13 | ``` 14 | Usage: tensorflow_model_server [--port=8500] [--enable_batching] [--model_name=my_name] --model_base_path=/path/to/export 15 | ``` 16 | 17 | ## Usage 18 | 19 | 1. Export the TensorFlow models in local host. 20 | 2. Run server with `./tensorflow_model_server --port=9000 --model_name=cancer --model_base_path=/tmp/cancer_model`. 21 | 22 | It is possible to run with docker container. Notice that the docker image is not public yet but easy to implement. 23 | 24 | ``` 25 | sudo docker run -d -p 9000:9000 -v /tmp/cancer_model:/tmp/cancer_model docker.d.xiaomi.net/cloud-ml/model-tensorflow-cpu:0.11.0 /model_service.py cancer /tmp/cancer_model 26 | ``` 27 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | This is the general tool to convert CSV file to TFRecords file. 4 | 5 | ## Cancer 6 | 7 | The example data in [cancer.csv](cancer.csv) looks like these. 8 | 9 | From [](https://github.com/mark-watson/cancer-deep-learning-model) 10 | 11 | ``` 12 | 3,7,7,4,4,9,4,8,1,1 13 | 1,1,1,1,2,1,2,1,1,0 14 | 4,1,1,3,2,1,3,1,1,0 15 | 7,8,7,2,4,8,3,8,2,1 16 | 9,5,8,1,2,3,2,1,5,1 17 | ``` 18 | 19 | The first 9th data are features and the last one is label. 20 | 21 | ``` 22 | - 0 Clump Thickness 1 - 10 23 | - 1 Uniformity of Cell Size 1 - 10 24 | - 2 Uniformity of Cell Shape 1 - 10 25 | - 3 Marginal Adhesion 1 - 10 26 | - 4 Single Epithelial Cell Size 1 - 10 27 | - 5 Bare Nuclei 1 - 10 28 | - 6 Bland Chromatin 1 - 10 29 | - 7 Normal Nucleoli 1 - 10 30 | - 8 Mitoses 1 - 10 31 | - 9 Class (0 for benign, 1 for malignant) 32 | ``` 33 | 34 | ## Usage 35 | 36 | Convert CSV file to TFRecords file with this script. 37 | 38 | ``` 39 | python convert_cancer_to_tfrecords.py 40 | ``` 41 | 42 | To verify the TFRecords, you can iterate and print each example with this script. 43 | 44 | ``` 45 | python print_cancer_tfrecords.py 46 | ``` 47 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/a8a/a8a_test.libsvm.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/a8a/a8a_test.libsvm.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/a8a/a8a_train.libsvm.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/a8a/a8a_train.libsvm.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/a8a/print_libsvm_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def print_tfrecords(input_filename): 8 | max_print_number = 100 9 | current_print_number = 0 10 | 11 | for serialized_example in tf.python_io.tf_record_iterator(input_filename): 12 | # Get serialized example from file 13 | example = tf.train.Example() 14 | example.ParseFromString(serialized_example) 15 | label = example.features.feature["label"].float_list.value 16 | ids = example.features.feature["ids"].int64_list.value 17 | values = example.features.feature["values"].float_list.value 18 | print("Number: {}, label: {}, features: {}".format( 19 | current_print_number, label, " ".join([str(id) + ":" + str( 20 | value) for id, value in zip(ids, values)]))) 21 | 22 | # Return when reaching max print number 23 | current_print_number += 1 24 | if current_print_number > max_print_number: 25 | exit() 26 | 27 | 28 | def main(): 29 | current_path = os.getcwd() 30 | tfrecords_file_name = "a8a_train.libsvm.tfrecords" 31 | input_filename = os.path.join(current_path, tfrecords_file_name) 32 | print_tfrecords(input_filename) 33 | 34 | 35 | if __name__ == "__main__": 36 | main() 37 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/a8a_test.libsvm: -------------------------------------------------------------------------------- 1 | 0 5:1 6:1 17:1 21:1 35:1 40:1 53:1 63:1 71:1 73:1 74:1 76:1 80:1 83:1 2 | 1 5:1 7:1 17:1 22:1 36:1 40:1 51:1 63:1 67:1 73:1 74:1 76:1 81:1 83:1 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/boston_housing/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Data 3 | 4 | The files are from https://inclass.kaggle.com/c/boston-housing . 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/boston_housing/print_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def print_tfrecords(input_filename): 8 | max_print_number = 100 9 | current_print_number = 0 10 | 11 | for serialized_example in tf.python_io.tf_record_iterator(input_filename): 12 | # Get serialized example from file 13 | example = tf.train.Example() 14 | example.ParseFromString(serialized_example) 15 | label = example.features.feature["label"].float_list.value 16 | features = example.features.feature["features"].float_list.value 17 | print("Number: {}, label: {}, features: {}".format(current_print_number, 18 | label, features)) 19 | 20 | # Return when reaching max print number 21 | current_print_number += 1 22 | if current_print_number > max_print_number: 23 | exit() 24 | 25 | 26 | def main(): 27 | current_path = os.getcwd() 28 | tfrecords_file_name = "train.csv.tfrecords" 29 | input_filename = os.path.join(current_path, tfrecords_file_name) 30 | print_tfrecords(input_filename) 31 | 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/boston_housing/train.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/boston_housing/train.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/cancer/cancer_test.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/cancer/cancer_test.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/cancer/cancer_train.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/cancer/cancer_train.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/cancer/generate_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def generate_tfrecords(input_filename, output_filename): 8 | print("Start to convert {} to {}".format(input_filename, output_filename)) 9 | writer = tf.python_io.TFRecordWriter(output_filename) 10 | 11 | for line in open(input_filename, "r"): 12 | data = line.split(",") 13 | label = float(data[9]) 14 | features = [float(i) for i in data[:9]] 15 | 16 | example = tf.train.Example(features=tf.train.Features(feature={ 17 | "label": 18 | tf.train.Feature(float_list=tf.train.FloatList(value=[label])), 19 | "features": 20 | tf.train.Feature(float_list=tf.train.FloatList(value=features)), 21 | })) 22 | writer.write(example.SerializeToString()) 23 | 24 | writer.close() 25 | print("Successfully convert {} to {}".format(input_filename, 26 | output_filename)) 27 | 28 | 29 | def main(): 30 | current_path = os.getcwd() 31 | for filename in os.listdir(current_path): 32 | if filename.startswith("") and filename.endswith(".csv"): 33 | generate_tfrecords(filename, filename + ".tfrecords") 34 | 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/cancer/print_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def print_tfrecords(input_filename): 8 | max_print_number = 100 9 | current_print_number = 0 10 | 11 | for serialized_example in tf.python_io.tf_record_iterator(input_filename): 12 | # Get serialized example from file 13 | example = tf.train.Example() 14 | example.ParseFromString(serialized_example) 15 | label = example.features.feature["label"].float_list.value 16 | features = example.features.feature["features"].float_list.value 17 | print("Number: {}, label: {}, features: {}".format(current_print_number, 18 | label, features)) 19 | 20 | # Return when reaching max print number 21 | current_print_number += 1 22 | if current_print_number > max_print_number: 23 | exit() 24 | 25 | 26 | def main(): 27 | current_path = os.getcwd() 28 | tfrecords_file_name = "cancer_train.csv.tfrecords" 29 | input_filename = os.path.join(current_path, tfrecords_file_name) 30 | print_tfrecords(input_filename) 31 | 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/cancer_test.csv: -------------------------------------------------------------------------------- 1 | 1,2,3,4,5,6,7,8,9,1 2 | 1,1,1,1,1,1,1,1,1,1 3 | 9,8,7,6,5,4,3,2,1,1 4 | 9,9,9,9,9,9,9,9,9,1 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/iris/generate_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def generate_tfrecords(input_filename, output_filename): 8 | print("Start to convert {} to {}".format(input_filename, output_filename)) 9 | writer = tf.python_io.TFRecordWriter(output_filename) 10 | 11 | for line in open(input_filename, "r"): 12 | data = line.split(",") 13 | label = float(data[0]) 14 | features = [float(i) for i in data[1:]] 15 | 16 | example = tf.train.Example(features=tf.train.Features(feature={ 17 | "label": 18 | tf.train.Feature(float_list=tf.train.FloatList(value=[label])), 19 | "features": 20 | tf.train.Feature(float_list=tf.train.FloatList(value=features)), 21 | })) 22 | writer.write(example.SerializeToString()) 23 | 24 | writer.close() 25 | print("Successfully convert {} to {}".format(input_filename, 26 | output_filename)) 27 | 28 | 29 | def main(): 30 | current_path = os.getcwd() 31 | for filename in os.listdir(current_path): 32 | if filename.startswith("") and filename.endswith(".csv"): 33 | generate_tfrecords(filename, filename + ".tfrecords") 34 | 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/iris/iris_test.csv: -------------------------------------------------------------------------------- 1 | 2,7.2,3.6,6.1,2.5 2 | 2,6.7,3.3,5.7,2.5 3 | 0,5.0,3.0,1.6,0.2 4 | 2,5.7,2.5,5.0,2.0 5 | 0,4.4,2.9,1.4,0.2 6 | 0,5.0,3.5,1.6,0.6 7 | 1,5.1,2.5,3.0,1.1 8 | 1,5.8,2.7,3.9,1.2 9 | 1,5.6,3.0,4.5,1.5 10 | 2,7.6,3.0,6.6,2.1 11 | 2,6.5,3.0,5.8,2.2 12 | 1,5.7,2.6,3.5,1.0 13 | 1,6.1,2.8,4.0,1.3 14 | 0,5.0,3.3,1.4,0.2 15 | 2,5.8,2.7,5.1,1.9 16 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/iris/iris_test.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/iris/iris_test.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/iris/iris_train.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/iris/iris_train.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/iris/print_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def print_tfrecords(input_filename): 8 | max_print_number = 100 9 | current_print_number = 0 10 | 11 | for serialized_example in tf.python_io.tf_record_iterator(input_filename): 12 | # Get serialized example from file 13 | example = tf.train.Example() 14 | example.ParseFromString(serialized_example) 15 | label = example.features.feature["label"].float_list.value 16 | features = example.features.feature["features"].float_list.value 17 | print("Number: {}, label: {}, features: {}".format(current_print_number, 18 | label, features)) 19 | 20 | # Return when reaching max print number 21 | current_print_number += 1 22 | if current_print_number > max_print_number: 23 | exit() 24 | 25 | 26 | def main(): 27 | current_path = os.getcwd() 28 | tfrecords_file_name = "iris_train.csv.tfrecords" 29 | input_filename = os.path.join(current_path, tfrecords_file_name) 30 | print_tfrecords(input_filename) 31 | 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/lung/fa7a21165ae152b13def786e6afc3edf.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/lung/fa7a21165ae152b13def786e6afc3edf.dcm -------------------------------------------------------------------------------- /tensorflow_template_application/data/lung/fa7a21165ae152b13def786e6afc3edf.dcm.csv.tfrecords: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/data/lung/fa7a21165ae152b13def786e6afc3edf.dcm.csv.tfrecords -------------------------------------------------------------------------------- /tensorflow_template_application/data/lung/generate_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def generate_tfrecords(input_filename, output_filename): 8 | print("Start to convert {} to {}".format(input_filename, output_filename)) 9 | writer = tf.python_io.TFRecordWriter(output_filename) 10 | 11 | for line in open(input_filename, "r"): 12 | data = line.split(",") 13 | label = float(data[0]) 14 | features = [float(i) for i in data[1:]] 15 | 16 | example = tf.train.Example(features=tf.train.Features(feature={ 17 | "label": 18 | tf.train.Feature(float_list=tf.train.FloatList(value=[label])), 19 | "features": 20 | tf.train.Feature(float_list=tf.train.FloatList(value=features)), 21 | })) 22 | writer.write(example.SerializeToString()) 23 | 24 | writer.close() 25 | print("Successfully convert {} to {}".format(input_filename, 26 | output_filename)) 27 | 28 | 29 | def main(): 30 | current_path = os.getcwd() 31 | for filename in os.listdir(current_path): 32 | if filename.startswith("") and filename.endswith(".csv"): 33 | generate_tfrecords(filename, filename + ".tfrecords") 34 | 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /tensorflow_template_application/data/lung/print_csv_tfrecords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tensorflow as tf 4 | import os 5 | 6 | 7 | def print_tfrecords(input_filename): 8 | max_print_number = 100 9 | current_print_number = 0 10 | 11 | for serialized_example in tf.python_io.tf_record_iterator(input_filename): 12 | # Get serialized example from file 13 | example = tf.train.Example() 14 | example.ParseFromString(serialized_example) 15 | label = example.features.feature["label"].float_list.value 16 | features = example.features.feature["features"].float_list.value 17 | print("Number: {}, label: {}, features: {}".format(current_print_number, 18 | label, features)) 19 | 20 | # Return when reaching max print number 21 | current_print_number += 1 22 | if current_print_number > max_print_number: 23 | exit() 24 | 25 | 26 | def main(): 27 | current_path = os.getcwd() 28 | tfrecords_file_name = "fa7a21165ae152b13def786e6afc3edf.dcm.csv.tfrecords" 29 | input_filename = os.path.join(current_path, tfrecords_file_name) 30 | print_tfrecords(input_filename) 31 | 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /tensorflow_template_application/distributed/checkpoint/.gitkeeper: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/distributed/checkpoint/.gitkeeper -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/README.md: -------------------------------------------------------------------------------- 1 | # Golang Predict Client 2 | 3 | ## Introduction 4 | 5 | It is the predict client in golang for TensorFlow Serving. 6 | 7 | You can compile the project easily and change the inference data for your TensorFlow models. 8 | 9 | ## Usage 10 | 11 | Install `protoc` and requirements. 12 | 13 | ``` 14 | go get -u github.com/golang/protobuf/{proto,protoc-gen-go} 15 | go get -u google.golang.org/grpc 16 | ``` 17 | 18 | Generate protobuf files. 19 | 20 | ``` 21 | cd ./src/ 22 | 23 | ./generate_proto_files.sh 24 | ``` 25 | 26 | Compile the project. 27 | 28 | ``` 29 | # Setup $GOPATH 30 | 31 | go build -x 32 | ``` 33 | 34 | Run the predict client. 35 | 36 | ``` 37 | # For dense model 38 | ./src --model_name dense 39 | 40 | # For sparse model 41 | ./src --model_name sparse 42 | ``` 43 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "predict.proto"; 7 | 8 | // PredictionService provides access to machine-learned models loaded by 9 | // model_servers. 10 | service PredictionService { 11 | // Predict -- provides access to loaded TensorFlow model. 12 | rpc Predict(PredictRequest) returns (PredictResponse); 13 | } 14 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/contrib/ios_examples/.gitignore: -------------------------------------------------------------------------------- 1 | project.xcworkspace 2 | xcuserdata 3 | imagenet_comp_graph_label_strings.txt 4 | tensorflow_inception_graph.pb 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/contrib/makefile/.gitignore: -------------------------------------------------------------------------------- 1 | gen/ 2 | downloads/ 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/contrib/pi_examples/.gitignore: -------------------------------------------------------------------------------- 1 | tensorflow_inception_graph.pb 2 | imagenet_comp_graph_label_strings.txt 3 | tensorflow_inception_stripped.pb 4 | */gen/ 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/allocation_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "AllocationDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message AllocationDescription { 10 | // Total number of bytes requested 11 | int64 requested_bytes = 1; 12 | 13 | // Total number of bytes allocated if known 14 | int64 allocated_bytes = 2; 15 | 16 | // Name of the allocator used 17 | string allocator_name = 3; 18 | 19 | // Identifier of the allocated buffer if known 20 | int64 allocation_id = 4; 21 | 22 | // Set if this tensor only has one remaining reference 23 | bool has_single_reference = 5; 24 | 25 | // Address of the allocation. 26 | uint64 ptr = 6; 27 | }; 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/device_attributes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "DeviceAttributesProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message DeviceLocality { 10 | // Optional bus locality of device. Default value of 0 means 11 | // no specific locality. Specific localities are indexed from 1. 12 | int32 bus_id = 1; 13 | }; 14 | 15 | message DeviceAttributes { 16 | // Fully specified name of the device within a cluster. 17 | string name = 1; 18 | 19 | // String representation of device_type. 20 | string device_type = 2; 21 | 22 | // Memory capacity of device in bytes. 23 | int64 memory_limit = 4; 24 | 25 | // Platform-specific data about device that may be useful 26 | // for supporting efficient data transfers. 27 | DeviceLocality locality = 5; 28 | 29 | // A device is assigned a global unique number each time it is 30 | // initialized. "incarnation" should never be 0. 31 | fixed64 incarnation = 6; 32 | 33 | // String representation of the physical device that this device maps to. 34 | string physical_device_desc = 7; 35 | } 36 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/kernel_def.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "KernelDefProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/attr_value.proto"; 10 | 11 | message KernelDef { 12 | // Must match the name of an Op. 13 | string op = 1; 14 | 15 | // Type of device this kernel runs on. 16 | string device_type = 2; 17 | 18 | message AttrConstraint { 19 | // Name of an attr from the Op. 20 | string name = 1; 21 | 22 | // A list of values that this kernel supports for this attr. 23 | // Like OpDef.AttrDef.allowed_values, except for kernels instead of Ops. 24 | AttrValue allowed_values = 2; 25 | } 26 | repeated AttrConstraint constraint = 3; 27 | 28 | // Names of the Op's input_/output_args that reside in host memory 29 | // instead of device memory. 30 | repeated string host_memory_arg = 4; 31 | 32 | // This allows experimental kernels to be registered for an op that 33 | // won't be used unless the user specifies a "_kernel" attr with 34 | // value matching this. 35 | string label = 5; 36 | } 37 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ResourceHandleProto"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 10 | // not valid across executions, but can be serialized back and forth from within 11 | // a single run. 12 | message ResourceHandle { 13 | // Unique name for the device containing the resource. 14 | string device = 1; 15 | 16 | // Container in which this resource is placed. 17 | string container = 2; 18 | 19 | // Unique name of this resource. 20 | string name = 3; 21 | 22 | // Hash code for the type of the resource. Is only valid in the same device 23 | // and in the same execution. 24 | uint64 hash_code = 4; 25 | 26 | // For debug-only, the name of the type pointed to by this handle, if 27 | // available. 28 | string maybe_type_name = 5; 29 | }; 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/tensor_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/types.proto"; 10 | import "tensorflow/core/framework/tensor_shape.proto"; 11 | import "tensorflow/core/framework/allocation_description.proto"; 12 | 13 | message TensorDescription { 14 | // Data type of tensor elements 15 | DataType dtype = 1; 16 | 17 | // Shape of the tensor. 18 | TensorShapeProto shape = 2; 19 | 20 | // Information about the size and allocator used for the data 21 | AllocationDescription allocation_description = 4; 22 | }; 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/variable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VariableProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a Variable. 10 | message VariableDef { 11 | // Name of the variable tensor. 12 | string variable_name = 1; 13 | 14 | // Name of the initializer op. 15 | string initializer_name = 2; 16 | 17 | // Name of the snapshot tensor. 18 | string snapshot_name = 3; 19 | 20 | // Support for saving variables as slices of a larger variable. 21 | SaveSliceInfoDef save_slice_info_def = 4; 22 | } 23 | 24 | message SaveSliceInfoDef { 25 | // Name of the full variable of which this is a slice. 26 | string full_name = 1; 27 | // Shape of the full variable. 28 | repeated int32 full_shape = 2; 29 | // Offset of this variable into the full variable. 30 | repeated int32 var_offset = 3; 31 | // Shape of this variable. 32 | repeated int32 var_shape = 4; 33 | } 34 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/framework/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VersionsProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Version information for a piece of serialized data 10 | // 11 | // There are different types of versions for each type of data 12 | // (GraphDef, etc.), but they all have the same common shape 13 | // described here. 14 | // 15 | // Each consumer has "consumer" and "min_producer" versions (specified 16 | // elsewhere). A consumer is allowed to consume this data if 17 | // 18 | // producer >= min_producer 19 | // consumer >= min_consumer 20 | // consumer not in bad_consumers 21 | // 22 | message VersionDef { 23 | // The version of the code that produced this data. 24 | int32 producer = 1; 25 | 26 | // Any consumer below this version is not allowed to consume this data. 27 | int32 min_consumer = 2; 28 | 29 | // Specific consumer versions which are disallowed (e.g. due to bugs). 30 | repeated int32 bad_consumers = 3; 31 | }; 32 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/kernels/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ReaderBaseProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.kernels"; 8 | 9 | // For serializing and restoring the state of ReaderBase, see 10 | // reader_base.h for details. 11 | message ReaderBaseState { 12 | int64 work_started = 1; 13 | int64 work_finished = 2; 14 | int64 num_records_produced = 3; 15 | bytes current_work = 4; 16 | }; 17 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/protobuf/named_tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "NamedTensorProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/tensor.proto"; 10 | 11 | // A pair of tensor name and tensor values. 12 | message NamedTensorProto { 13 | // Name of the tensor. 14 | string name = 1; 15 | 16 | // The client can populate a TensorProto using a tensorflow::Tensor`, or 17 | // directly using the protobuf field accessors. 18 | // 19 | // The client specifies whether the returned tensor values should be 20 | // filled tensor fields (float_val, int_val, etc.) or encoded in a 21 | // compact form in tensor.tensor_content. 22 | TensorProto tensor = 2; 23 | } 24 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/protobuf/queue_runner.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "QueueRunnerProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/lib/core/error_codes.proto"; 10 | 11 | // Protocol buffer representing a QueueRunner. 12 | message QueueRunnerDef { 13 | // Queue name. 14 | string queue_name = 1; 15 | 16 | // A list of enqueue operations. 17 | repeated string enqueue_op_name = 2; 18 | 19 | // The operation to run to close the queue. 20 | string close_op_name = 3; 21 | 22 | // The operation to run to cancel the queue. 23 | string cancel_op_name = 4; 24 | 25 | // A list of exception types considered to signal a safely closed queue 26 | // if raised during enqueue operations. 27 | repeated error.Code queue_closed_exception_types = 5; 28 | } 29 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/protobuf/saved_model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "SavedModelProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/protobuf/meta_graph.proto"; 10 | 11 | // SavedModel is the high level serialization format for TensorFlow Models. 12 | // See [todo: doc links, similar to session_bundle] for more information. 13 | message SavedModel { 14 | // The schema version of the SavedModel instance. Used for versioning when 15 | // making future changes to the specification/implementation. Initial value 16 | // at release will be 1. 17 | int64 saved_model_schema_version = 1; 18 | 19 | // One or more MetaGraphs. 20 | repeated MetaGraphDef meta_graphs = 2; 21 | } 22 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/core/util/memmapped_file_system.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package tensorflow; 18 | option cc_enable_arenas = true; 19 | 20 | // A message that describes one region of memmapped file. 21 | message MemmappedFileSystemDirectoryElement { 22 | uint64 offset = 1; 23 | string name = 2; 24 | } 25 | 26 | // A directory of regions in a memmapped file. 27 | message MemmappedFileSystemDirectory { 28 | repeated MemmappedFileSystemDirectoryElement element = 1; 29 | } 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/examples/udacity/.gitignore: -------------------------------------------------------------------------------- 1 | notMNIST_large* 2 | notMNIST_small* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/go/genop/.gitignore: -------------------------------------------------------------------------------- 1 | # .pb.go files generated by generate.sh 2 | internal/proto/* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/go/op/.gitignore: -------------------------------------------------------------------------------- 1 | wrappers.go 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/python/framework/cpp_shape_inference.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow/core/framework/types.proto"; 7 | import "tensorflow/core/framework/tensor_shape.proto"; 8 | 9 | message CppShapeInferenceResult { 10 | TensorShapeProto shape = 1; 11 | TensorShapeProto handle_shape = 2; 12 | DataType handle_dtype = 3; 13 | } -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/python/training/checkpoint_state.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | // Protocol buffer representing the checkpoint state. 7 | // 8 | // TODO(touts): Add other attributes as needed. 9 | message CheckpointState { 10 | // Path to the most-recent model checkpoint. 11 | string model_checkpoint_path = 1; 12 | 13 | // Paths to all not-yet-deleted model checkpoints, sorted from oldest to 14 | // newest. 15 | // Note that the value of model_checkpoint_path should be the last item in 16 | // this list. 17 | repeated string all_model_checkpoint_paths = 2; 18 | } 19 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/python/util/protobuf/compare_test.proto: -------------------------------------------------------------------------------- 1 | // Test messages used in compare_test.py. 2 | syntax = "proto2"; 3 | 4 | package compare_test; 5 | option cc_enable_arenas = true; 6 | 7 | enum Enum { 8 | A = 0; 9 | B = 1; 10 | C = 2; 11 | } 12 | 13 | message Small { 14 | repeated string strings = 1; 15 | }; 16 | 17 | message Medium { 18 | repeated int32 int32s = 1; 19 | repeated Small smalls = 2; 20 | repeated group GroupA = 3 { 21 | repeated group GroupB = 4 { 22 | required string strings = 5; 23 | } 24 | } 25 | repeated float floats = 6; 26 | }; 27 | 28 | message Large { 29 | optional string string_ = 1; 30 | optional int64 int64_ = 2; 31 | optional float float_ = 3; 32 | optional bool bool_ = 4; 33 | optional Enum enum_ = 5; 34 | repeated int64 int64s = 6; 35 | optional Medium medium = 7; 36 | optional Small small = 8; 37 | optional double double_ = 9; 38 | optional WithMap with_map = 10; 39 | }; 40 | 41 | message Labeled { 42 | required int32 required = 1; 43 | optional int32 optional = 2; 44 | } 45 | 46 | message WithMap { 47 | map value_message = 1; 48 | map value_string = 2; 49 | } 50 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tensorboard/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory" : "components" 3 | } -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tensorboard/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | typings/* 3 | build/* 4 | dist/tf-tensorboard-demo.html 5 | 6 | # Since bower components are stored in the same directory as 7 | # tensorboard components, we ignore everything under components 8 | # except our own components which start with tf-. 9 | components/* 10 | # This rule should always be in sync with TF_COMPONENTS_TYPESCRIPT_GLOB 11 | # in gulpfile.js 12 | !components/tf-* 13 | !components/tf_* 14 | !components/vz-* 15 | !components/vz_* 16 | !components/index.html 17 | !components/BUILD 18 | # Ignore the sample graph files since they are too large to 19 | # be in the repo. 20 | components/tf-graph/demo/tf_model_zoo/* 21 | 22 | # All standalone code for TensorBoard components should be written in 23 | # typescript, and the compiled javascript code should be ignored. 24 | components/tf-*/**/*.js 25 | components/tf_*/**/*.js 26 | components/vz-*/**/*.js 27 | components/vz_*/**/*.js 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tools/ci_build/install/.bazelrc: -------------------------------------------------------------------------------- 1 | # Running bazel inside a `docker build` command causes trouble, cf: 2 | # https://github.com/bazelbuild/bazel/issues/134 3 | # The easiest solution is to set up a bazelrc file forcing --batch. 4 | startup --batch 5 | 6 | # Similarly, we need to workaround sandboxing issues: 7 | # https://github.com/bazelbuild/bazel/issues/418 8 | build --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone 9 | test --spawn_strategy=standalone 10 | 11 | # Force bazel output to use colors (good for jenkins) and print useful errors. 12 | common --color=yes 13 | 14 | # Configure tests - increase timeout, print errors and timeout warnings 15 | test --verbose_failures --test_output=errors --test_verbose_timeout_warnings 16 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tools/git/.gitignore: -------------------------------------------------------------------------------- 1 | gen 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tools/swig/.gitignore: -------------------------------------------------------------------------------- 1 | swig_path 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tools/tfprof/tfprof_log.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | message OpLogEntry { 6 | // op name. 7 | optional string name = 1; 8 | // float_ops is filled by tfprof Python API when called. It requires the 9 | // op has RegisterStatistics defined. Currently, Conv2D, MatMul, etc, are 10 | // implemented. 11 | optional int64 float_ops = 2; 12 | // User can define extra op type information for an op. This allows the user 13 | // to select a group of ops precisely using op_type as a key. 14 | repeated string types = 3; 15 | } 16 | 17 | message OpLog { 18 | repeated OpLogEntry log_entries = 1; 19 | } -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_golang_files/tensorflow/tools/tfprof/tfprof_options.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | // Refers to tfprof_options.h/cc for documentation. 6 | // Only used to pass tfprof options from Python to C++. 7 | message OptionsProto { 8 | optional int64 max_depth = 1; 9 | optional int64 min_bytes = 2; 10 | optional int64 min_micros = 3; 11 | optional int64 min_params = 4; 12 | optional int64 min_float_ops = 5; 13 | repeated string device_regexes = 6; 14 | optional string order_by = 7; 15 | repeated string account_type_regexes = 8; 16 | repeated string start_name_regexes = 9; 17 | repeated string trim_name_regexes = 10; 18 | repeated string show_name_regexes = 11; 19 | repeated string hide_name_regexes = 12; 20 | optional bool account_displayed_op_only = 13; 21 | repeated string select = 14; 22 | optional bool viz = 15; 23 | optional string dump_to_file = 16; 24 | } -------------------------------------------------------------------------------- /tensorflow_template_application/golang_predict_client/src/generate_proto_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | # install gRPC and protoc plugin for Go, see http://www.grpc.io/docs/quickstart/go.html#generate-grpc-code 5 | mkdir tensorflow tensorflow_serving 6 | protoc -I generate_golang_files/ generate_golang_files/*.proto --go_out=plugins=grpc:tensorflow_serving 7 | protoc -I generate_golang_files/ generate_golang_files/tensorflow/core/framework/* --go_out=plugins=grpc:. -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/README.md: -------------------------------------------------------------------------------- 1 | # Cancer Predictiton Service 2 | 3 | This is the HTTP service to load TensorFlow model and inference to predict cancer. 4 | 5 | ## Usage 6 | 7 | 8 | Run HTTP server with [Django] and use HTTP client with curl, postman or [bat](https://github.com/astaxie/bat) for test. 9 | 10 | ``` 11 | ./manage.py runserver 0.0.0.0:8000 12 | ``` 13 | 14 | ### Inference to predict cancel 15 | 16 | ``` 17 | bat post http://127.0.0.1:8000/cancer_predict/predict/ cancer_features="10,10,10,8,6,1,8,9,1;6,2,1,1,1,1,7,1,1" 18 | ``` 19 | 20 | ![](./images/predict.png) 21 | 22 | ### Online learning to train 23 | 24 | ``` 25 | bat post http://127.0.0.1:8000/cancer_predict/online_train/ cancer_features_and_labels="10,10,10,8,6,1,8,9,1,1;6,2,1,1,1,1,7,1,1,0" 26 | ``` 27 | 28 | ![](./images/online_train.png) 29 | 30 | ## Implementation 31 | 32 | ``` 33 | django-admin startproject restful_server 34 | 35 | python manage.py startapp cancer_predict 36 | 37 | # Add customized urls and views. 38 | ``` 39 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/cancer_predict/__init__.py -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/apps.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class CancerPredictConfig(AppConfig): 7 | name = 'cancer_predict' 8 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/cancer_predict/migrations/__init__.py -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.db import models 4 | 5 | # Create your models here. 6 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/cancer_predict/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from . import views 3 | 4 | urlpatterns = [ 5 | url(r'^predict/', views.predict, 6 | name='predict'), 7 | url(r'^$', views.index, name='index'), 8 | ] 9 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/db.sqlite3 -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/images/online_train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/images/online_train.png -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/images/predict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/images/predict.png -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", 7 | "restful_server.settings") 8 | 9 | from django.core.management import execute_from_command_line 10 | 11 | execute_from_command_line(sys.argv) 12 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/predict.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bat post http://127.0.0.1:8000/cancer_predict/predict/ < predict_examples.json 4 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/predict_examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": [1.0, 2.0], 3 | "features": [[10,10,10,8,6,1,8,9,1], [6,2,1,1,1,1,7,1,1]] 4 | } 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/restful_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/http_service/restful_server/__init__.py -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/restful_server/urls.py: -------------------------------------------------------------------------------- 1 | """restful_server URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url 17 | from django.contrib import admin 18 | 19 | from django.conf.urls import include, url 20 | 21 | urlpatterns = [ 22 | url(r'^cancer_predict/', include('cancer_predict.urls')), 23 | url(r'^admin/', admin.site.urls), 24 | ] 25 | -------------------------------------------------------------------------------- /tensorflow_template_application/http_service/restful_server/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for restful_server project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", 15 | "restful_server.settings") 16 | 17 | application = get_wsgi_application() 18 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | mvn clean install -DskipTests 7 | 8 | mvn exec:java -Dexec.mainClass="com.tobe.PredictClient" -Dexec.args="127.0.0.1 9000 cancer 1" 9 | 10 | mvn compile exec:java -Dexec.mainClass="com.tobe.SparsePredictClient" -Dexec.args="127.0.0.1 9000 sparse 1" 11 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "predict.proto"; 7 | 8 | // PredictionService provides access to machine-learned models loaded by 9 | // model_servers. 10 | service PredictionService { 11 | // Predict -- provides access to loaded TensorFlow model. 12 | rpc Predict(PredictRequest) returns (PredictResponse); 13 | } 14 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/contrib/ios_examples/.gitignore: -------------------------------------------------------------------------------- 1 | project.xcworkspace 2 | xcuserdata 3 | imagenet_comp_graph_label_strings.txt 4 | tensorflow_inception_graph.pb 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/contrib/makefile/.gitignore: -------------------------------------------------------------------------------- 1 | gen/ 2 | downloads/ 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/contrib/pi_examples/.gitignore: -------------------------------------------------------------------------------- 1 | tensorflow_inception_graph.pb 2 | imagenet_comp_graph_label_strings.txt 3 | tensorflow_inception_stripped.pb 4 | */gen/ 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/allocation_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "AllocationDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message AllocationDescription { 10 | // Total number of bytes requested 11 | int64 requested_bytes = 1; 12 | 13 | // Total number of bytes allocated if known 14 | int64 allocated_bytes = 2; 15 | 16 | // Name of the allocator used 17 | string allocator_name = 3; 18 | 19 | // Identifier of the allocated buffer if known 20 | int64 allocation_id = 4; 21 | 22 | // Set if this tensor only has one remaining reference 23 | bool has_single_reference = 5; 24 | 25 | // Address of the allocation. 26 | uint64 ptr = 6; 27 | }; 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/device_attributes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "DeviceAttributesProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message DeviceLocality { 10 | // Optional bus locality of device. Default value of 0 means 11 | // no specific locality. Specific localities are indexed from 1. 12 | int32 bus_id = 1; 13 | }; 14 | 15 | message DeviceAttributes { 16 | // Fully specified name of the device within a cluster. 17 | string name = 1; 18 | 19 | // String representation of device_type. 20 | string device_type = 2; 21 | 22 | // Memory capacity of device in bytes. 23 | int64 memory_limit = 4; 24 | 25 | // Platform-specific data about device that may be useful 26 | // for supporting efficient data transfers. 27 | DeviceLocality locality = 5; 28 | 29 | // A device is assigned a global unique number each time it is 30 | // initialized. "incarnation" should never be 0. 31 | fixed64 incarnation = 6; 32 | 33 | // String representation of the physical device that this device maps to. 34 | string physical_device_desc = 7; 35 | } 36 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/kernel_def.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "KernelDefProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/attr_value.proto"; 10 | 11 | message KernelDef { 12 | // Must match the name of an Op. 13 | string op = 1; 14 | 15 | // Type of device this kernel runs on. 16 | string device_type = 2; 17 | 18 | message AttrConstraint { 19 | // Name of an attr from the Op. 20 | string name = 1; 21 | 22 | // A list of values that this kernel supports for this attr. 23 | // Like OpDef.AttrDef.allowed_values, except for kernels instead of Ops. 24 | AttrValue allowed_values = 2; 25 | } 26 | repeated AttrConstraint constraint = 3; 27 | 28 | // Names of the Op's input_/output_args that reside in host memory 29 | // instead of device memory. 30 | repeated string host_memory_arg = 4; 31 | 32 | // This allows experimental kernels to be registered for an op that 33 | // won't be used unless the user specifies a "_kernel" attr with 34 | // value matching this. 35 | string label = 5; 36 | } 37 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ResourceHandleProto"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 10 | // not valid across executions, but can be serialized back and forth from within 11 | // a single run. 12 | message ResourceHandle { 13 | // Unique name for the device containing the resource. 14 | string device = 1; 15 | 16 | // Container in which this resource is placed. 17 | string container = 2; 18 | 19 | // Unique name of this resource. 20 | string name = 3; 21 | 22 | // Hash code for the type of the resource. Is only valid in the same device 23 | // and in the same execution. 24 | uint64 hash_code = 4; 25 | 26 | // For debug-only, the name of the type pointed to by this handle, if 27 | // available. 28 | string maybe_type_name = 5; 29 | }; 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/tensor_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/types.proto"; 10 | import "tensorflow/core/framework/tensor_shape.proto"; 11 | import "tensorflow/core/framework/allocation_description.proto"; 12 | 13 | message TensorDescription { 14 | // Data type of tensor elements 15 | DataType dtype = 1; 16 | 17 | // Shape of the tensor. 18 | TensorShapeProto shape = 2; 19 | 20 | // Information about the size and allocator used for the data 21 | AllocationDescription allocation_description = 4; 22 | }; 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/variable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VariableProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a Variable. 10 | message VariableDef { 11 | // Name of the variable tensor. 12 | string variable_name = 1; 13 | 14 | // Name of the initializer op. 15 | string initializer_name = 2; 16 | 17 | // Name of the snapshot tensor. 18 | string snapshot_name = 3; 19 | 20 | // Support for saving variables as slices of a larger variable. 21 | SaveSliceInfoDef save_slice_info_def = 4; 22 | } 23 | 24 | message SaveSliceInfoDef { 25 | // Name of the full variable of which this is a slice. 26 | string full_name = 1; 27 | // Shape of the full variable. 28 | repeated int32 full_shape = 2; 29 | // Offset of this variable into the full variable. 30 | repeated int32 var_offset = 3; 31 | // Shape of this variable. 32 | repeated int32 var_shape = 4; 33 | } 34 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/framework/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VersionsProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Version information for a piece of serialized data 10 | // 11 | // There are different types of versions for each type of data 12 | // (GraphDef, etc.), but they all have the same common shape 13 | // described here. 14 | // 15 | // Each consumer has "consumer" and "min_producer" versions (specified 16 | // elsewhere). A consumer is allowed to consume this data if 17 | // 18 | // producer >= min_producer 19 | // consumer >= min_consumer 20 | // consumer not in bad_consumers 21 | // 22 | message VersionDef { 23 | // The version of the code that produced this data. 24 | int32 producer = 1; 25 | 26 | // Any consumer below this version is not allowed to consume this data. 27 | int32 min_consumer = 2; 28 | 29 | // Specific consumer versions which are disallowed (e.g. due to bugs). 30 | repeated int32 bad_consumers = 3; 31 | }; 32 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/kernels/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ReaderBaseProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.kernels"; 8 | 9 | // For serializing and restoring the state of ReaderBase, see 10 | // reader_base.h for details. 11 | message ReaderBaseState { 12 | int64 work_started = 1; 13 | int64 work_finished = 2; 14 | int64 num_records_produced = 3; 15 | bytes current_work = 4; 16 | }; 17 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/protobuf/named_tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "NamedTensorProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/tensor.proto"; 10 | 11 | // A pair of tensor name and tensor values. 12 | message NamedTensorProto { 13 | // Name of the tensor. 14 | string name = 1; 15 | 16 | // The client can populate a TensorProto using a tensorflow::Tensor`, or 17 | // directly using the protobuf field accessors. 18 | // 19 | // The client specifies whether the returned tensor values should be 20 | // filled tensor fields (float_val, int_val, etc.) or encoded in a 21 | // compact form in tensor.tensor_content. 22 | TensorProto tensor = 2; 23 | } 24 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/protobuf/queue_runner.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "QueueRunnerProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/lib/core/error_codes.proto"; 10 | 11 | // Protocol buffer representing a QueueRunner. 12 | message QueueRunnerDef { 13 | // Queue name. 14 | string queue_name = 1; 15 | 16 | // A list of enqueue operations. 17 | repeated string enqueue_op_name = 2; 18 | 19 | // The operation to run to close the queue. 20 | string close_op_name = 3; 21 | 22 | // The operation to run to cancel the queue. 23 | string cancel_op_name = 4; 24 | 25 | // A list of exception types considered to signal a safely closed queue 26 | // if raised during enqueue operations. 27 | repeated error.Code queue_closed_exception_types = 5; 28 | } 29 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/protobuf/saved_model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "SavedModelProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/protobuf/meta_graph.proto"; 10 | 11 | // SavedModel is the high level serialization format for TensorFlow Models. 12 | // See [todo: doc links, similar to session_bundle] for more information. 13 | message SavedModel { 14 | // The schema version of the SavedModel instance. Used for versioning when 15 | // making future changes to the specification/implementation. Initial value 16 | // at release will be 1. 17 | int64 saved_model_schema_version = 1; 18 | 19 | // One or more MetaGraphs. 20 | repeated MetaGraphDef meta_graphs = 2; 21 | } 22 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/core/util/memmapped_file_system.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package tensorflow; 18 | option cc_enable_arenas = true; 19 | 20 | // A message that describes one region of memmapped file. 21 | message MemmappedFileSystemDirectoryElement { 22 | uint64 offset = 1; 23 | string name = 2; 24 | } 25 | 26 | // A directory of regions in a memmapped file. 27 | message MemmappedFileSystemDirectory { 28 | repeated MemmappedFileSystemDirectoryElement element = 1; 29 | } 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/examples/udacity/.gitignore: -------------------------------------------------------------------------------- 1 | notMNIST_large* 2 | notMNIST_small* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/go/genop/.gitignore: -------------------------------------------------------------------------------- 1 | # .pb.go files generated by generate.sh 2 | internal/proto/* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/go/op/.gitignore: -------------------------------------------------------------------------------- 1 | wrappers.go 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/python/framework/cpp_shape_inference.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow/core/framework/types.proto"; 7 | import "tensorflow/core/framework/tensor_shape.proto"; 8 | 9 | message CppShapeInferenceResult { 10 | TensorShapeProto shape = 1; 11 | TensorShapeProto handle_shape = 2; 12 | DataType handle_dtype = 3; 13 | } -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/python/training/checkpoint_state.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | // Protocol buffer representing the checkpoint state. 7 | // 8 | // TODO(touts): Add other attributes as needed. 9 | message CheckpointState { 10 | // Path to the most-recent model checkpoint. 11 | string model_checkpoint_path = 1; 12 | 13 | // Paths to all not-yet-deleted model checkpoints, sorted from oldest to 14 | // newest. 15 | // Note that the value of model_checkpoint_path should be the last item in 16 | // this list. 17 | repeated string all_model_checkpoint_paths = 2; 18 | } 19 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/python/util/protobuf/compare_test.proto: -------------------------------------------------------------------------------- 1 | // Test messages used in compare_test.py. 2 | syntax = "proto2"; 3 | 4 | package compare_test; 5 | option cc_enable_arenas = true; 6 | 7 | enum Enum { 8 | A = 0; 9 | B = 1; 10 | C = 2; 11 | } 12 | 13 | message Small { 14 | repeated string strings = 1; 15 | }; 16 | 17 | message Medium { 18 | repeated int32 int32s = 1; 19 | repeated Small smalls = 2; 20 | repeated group GroupA = 3 { 21 | repeated group GroupB = 4 { 22 | required string strings = 5; 23 | } 24 | } 25 | repeated float floats = 6; 26 | }; 27 | 28 | message Large { 29 | optional string string_ = 1; 30 | optional int64 int64_ = 2; 31 | optional float float_ = 3; 32 | optional bool bool_ = 4; 33 | optional Enum enum_ = 5; 34 | repeated int64 int64s = 6; 35 | optional Medium medium = 7; 36 | optional Small small = 8; 37 | optional double double_ = 9; 38 | optional WithMap with_map = 10; 39 | }; 40 | 41 | message Labeled { 42 | required int32 required = 1; 43 | optional int32 optional = 2; 44 | } 45 | 46 | message WithMap { 47 | map value_message = 1; 48 | map value_string = 2; 49 | } 50 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tensorboard/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory" : "components" 3 | } -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tensorboard/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | typings/* 3 | build/* 4 | dist/tf-tensorboard-demo.html 5 | 6 | # Since bower components are stored in the same directory as 7 | # tensorboard components, we ignore everything under components 8 | # except our own components which start with tf-. 9 | components/* 10 | # This rule should always be in sync with TF_COMPONENTS_TYPESCRIPT_GLOB 11 | # in gulpfile.js 12 | !components/tf-* 13 | !components/tf_* 14 | !components/vz-* 15 | !components/vz_* 16 | !components/index.html 17 | !components/BUILD 18 | # Ignore the sample graph files since they are too large to 19 | # be in the repo. 20 | components/tf-graph/demo/tf_model_zoo/* 21 | 22 | # All standalone code for TensorBoard components should be written in 23 | # typescript, and the compiled javascript code should be ignored. 24 | components/tf-*/**/*.js 25 | components/tf_*/**/*.js 26 | components/vz-*/**/*.js 27 | components/vz_*/**/*.js 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tools/ci_build/install/.bazelrc: -------------------------------------------------------------------------------- 1 | # Running bazel inside a `docker build` command causes trouble, cf: 2 | # https://github.com/bazelbuild/bazel/issues/134 3 | # The easiest solution is to set up a bazelrc file forcing --batch. 4 | startup --batch 5 | 6 | # Similarly, we need to workaround sandboxing issues: 7 | # https://github.com/bazelbuild/bazel/issues/418 8 | build --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone 9 | test --spawn_strategy=standalone 10 | 11 | # Force bazel output to use colors (good for jenkins) and print useful errors. 12 | common --color=yes 13 | 14 | # Configure tests - increase timeout, print errors and timeout warnings 15 | test --verbose_failures --test_output=errors --test_verbose_timeout_warnings 16 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tools/git/.gitignore: -------------------------------------------------------------------------------- 1 | gen 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tools/swig/.gitignore: -------------------------------------------------------------------------------- 1 | swig_path 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tools/tfprof/tfprof_log.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | message OpLogEntry { 6 | // op name. 7 | optional string name = 1; 8 | // float_ops is filled by tfprof Python API when called. It requires the 9 | // op has RegisterStatistics defined. Currently, Conv2D, MatMul, etc, are 10 | // implemented. 11 | optional int64 float_ops = 2; 12 | // User can define extra op type information for an op. This allows the user 13 | // to select a group of ops precisely using op_type as a key. 14 | repeated string types = 3; 15 | } 16 | 17 | message OpLog { 18 | repeated OpLogEntry log_entries = 1; 19 | } -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/proto/tensorflow/tools/tfprof/tfprof_options.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | // Refers to tfprof_options.h/cc for documentation. 6 | // Only used to pass tfprof options from Python to C++. 7 | message OptionsProto { 8 | optional int64 max_depth = 1; 9 | optional int64 min_bytes = 2; 10 | optional int64 min_micros = 3; 11 | optional int64 min_params = 4; 12 | optional int64 min_float_ops = 5; 13 | repeated string device_regexes = 6; 14 | optional string order_by = 7; 15 | repeated string account_type_regexes = 8; 16 | repeated string start_name_regexes = 9; 17 | repeated string trim_name_regexes = 10; 18 | repeated string show_name_regexes = 11; 19 | repeated string hide_name_regexes = 12; 20 | optional bool account_displayed_op_only = 13; 21 | repeated string select = 14; 22 | optional bool viz = 15; 23 | optional string dump_to_file = 16; 24 | } -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/src/main/scala/com/tobe/client/ScalaDensePredictClient.scala: -------------------------------------------------------------------------------- 1 | package com.tobe.client 2 | 3 | object ScalaDensePredictClient { 4 | 5 | def main(args: Array[String]): Unit = { 6 | 7 | System.out.println("Start scala project") 8 | 9 | var host: String = "127.0.0.1" 10 | var port: Int = 9000 11 | var modelName: String = "dense" 12 | var modelVersion: Long = 1 13 | 14 | // TODO: String to int doesn't work 15 | // Parse command-line arguments 16 | if (args.length == 4) { 17 | host = args(0) 18 | //port = args(1).toInt 19 | modelName = args(2) 20 | //modelVersion = args(3).toLong 21 | } 22 | 23 | // Create dense predict client 24 | val client: DensePredictClient = new DensePredictClient(host, port) 25 | 26 | // Run predict client to send request 27 | client.predict_example(modelName, modelVersion) 28 | 29 | System.out.println("End of predict client") 30 | // TODO: Exit the project well 31 | System.exit(0) 32 | 33 | } 34 | 35 | } -------------------------------------------------------------------------------- /tensorflow_template_application/java_predict_client/update_tensorflow_proto_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # Clone TensorFlow project in proto directory 7 | 8 | ls ./tensorflow/**/*[a-n] | xargs rm 9 | ls ./tensorflow/**/*[p-z] | xargs rm 10 | ls ./tensorflow/**/*[A-Z] | xargs rm 11 | ls ./tensorflow/**/*[0-9] | xargs rm 12 | ls ./tensorflow/**/*.go | xargs rm 13 | 14 | find . -type d -empty -print | xargs rmdir 15 | find . -type d -empty -print | xargs rmdir 16 | find . -type d -empty -print | xargs rmdir 17 | find . -type d -empty -print | xargs rmdir 18 | find . -type d -empty -print | xargs rmdir 19 | -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | The minimal TensorFlow application for benchmarking. 4 | 5 | ## Start predict server 6 | 7 | ``` 8 | ./tensorflow_model_server --port=9000 --model_name=minimal --model_base_path=./model 9 | ``` 10 | 11 | ## Start predict client 12 | 13 | ``` 14 | ./predict_client.py --host 127.0.0.1 --port 9000 --model_name minial --model_version 1 15 | ``` 16 | 17 | ``` 18 | cloudml models predict -n minial -s 127.0.0.1:9000 -f ./data.json 19 | ``` 20 | 21 | ## Benchmark 22 | 23 | ### Test local predict 24 | 25 | ``` 26 | ./benchmark_predict.py --benchmark_test_number 10 --benchmark_batch_size 1 27 | ``` 28 | 29 | ### Test python gRPC client 30 | 31 | ``` 32 | ./benchmark_qps.py --host 127.0.0.1 --port 9000 --model_name minial --model_version 1 --benchmark_batch_size 100 --benchmark_test_number 10000 33 | ``` 34 | -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "features_dtype": "float32", 3 | "features": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 4 | } 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/model/00000001/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "export" 2 | all_model_checkpoint_paths: "export" 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/model/00000001/export.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/minimal_model/model/00000001/export.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/model/00000001/export.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/minimal_model/model/00000001/export.index -------------------------------------------------------------------------------- /tensorflow_template_application/minimal_model/model/00000001/export.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/minimal_model/model/00000001/export.meta -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/generate_proto_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | python -m grpc.tools.protoc -I./ --python_out=.. --grpc_python_out=.. ./*.proto 7 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "predict.proto"; 7 | 8 | // PredictionService provides access to machine-learned models loaded by 9 | // model_servers. 10 | service PredictionService { 11 | // Predict -- provides access to loaded TensorFlow model. 12 | rpc Predict(PredictRequest) returns (PredictResponse); 13 | } 14 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/contrib/ios_examples/.gitignore: -------------------------------------------------------------------------------- 1 | project.xcworkspace 2 | xcuserdata 3 | imagenet_comp_graph_label_strings.txt 4 | tensorflow_inception_graph.pb 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/contrib/makefile/.gitignore: -------------------------------------------------------------------------------- 1 | gen/ 2 | downloads/ 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/contrib/pi_examples/.gitignore: -------------------------------------------------------------------------------- 1 | tensorflow_inception_graph.pb 2 | imagenet_comp_graph_label_strings.txt 3 | tensorflow_inception_stripped.pb 4 | */gen/ 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/allocation_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "AllocationDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message AllocationDescription { 10 | // Total number of bytes requested 11 | int64 requested_bytes = 1; 12 | 13 | // Total number of bytes allocated if known 14 | int64 allocated_bytes = 2; 15 | 16 | // Name of the allocator used 17 | string allocator_name = 3; 18 | 19 | // Identifier of the allocated buffer if known 20 | int64 allocation_id = 4; 21 | 22 | // Set if this tensor only has one remaining reference 23 | bool has_single_reference = 5; 24 | 25 | // Address of the allocation. 26 | uint64 ptr = 6; 27 | }; 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/device_attributes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "DeviceAttributesProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | message DeviceLocality { 10 | // Optional bus locality of device. Default value of 0 means 11 | // no specific locality. Specific localities are indexed from 1. 12 | int32 bus_id = 1; 13 | }; 14 | 15 | message DeviceAttributes { 16 | // Fully specified name of the device within a cluster. 17 | string name = 1; 18 | 19 | // String representation of device_type. 20 | string device_type = 2; 21 | 22 | // Memory capacity of device in bytes. 23 | int64 memory_limit = 4; 24 | 25 | // Platform-specific data about device that may be useful 26 | // for supporting efficient data transfers. 27 | DeviceLocality locality = 5; 28 | 29 | // A device is assigned a global unique number each time it is 30 | // initialized. "incarnation" should never be 0. 31 | fixed64 incarnation = 6; 32 | 33 | // String representation of the physical device that this device maps to. 34 | string physical_device_desc = 7; 35 | } 36 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/kernel_def.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "KernelDefProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/attr_value.proto"; 10 | 11 | message KernelDef { 12 | // Must match the name of an Op. 13 | string op = 1; 14 | 15 | // Type of device this kernel runs on. 16 | string device_type = 2; 17 | 18 | message AttrConstraint { 19 | // Name of an attr from the Op. 20 | string name = 1; 21 | 22 | // A list of values that this kernel supports for this attr. 23 | // Like OpDef.AttrDef.allowed_values, except for kernels instead of Ops. 24 | AttrValue allowed_values = 2; 25 | } 26 | repeated AttrConstraint constraint = 3; 27 | 28 | // Names of the Op's input_/output_args that reside in host memory 29 | // instead of device memory. 30 | repeated string host_memory_arg = 4; 31 | 32 | // This allows experimental kernels to be registered for an op that 33 | // won't be used unless the user specifies a "_kernel" attr with 34 | // value matching this. 35 | string label = 5; 36 | } 37 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ResourceHandleProto"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 10 | // not valid across executions, but can be serialized back and forth from within 11 | // a single run. 12 | message ResourceHandle { 13 | // Unique name for the device containing the resource. 14 | string device = 1; 15 | 16 | // Container in which this resource is placed. 17 | string container = 2; 18 | 19 | // Unique name of this resource. 20 | string name = 3; 21 | 22 | // Hash code for the type of the resource. Is only valid in the same device 23 | // and in the same execution. 24 | uint64 hash_code = 4; 25 | 26 | // For debug-only, the name of the type pointed to by this handle, if 27 | // available. 28 | string maybe_type_name = 5; 29 | }; 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/tensor_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorDescriptionProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/types.proto"; 10 | import "tensorflow/core/framework/tensor_shape.proto"; 11 | import "tensorflow/core/framework/allocation_description.proto"; 12 | 13 | message TensorDescription { 14 | // Data type of tensor elements 15 | DataType dtype = 1; 16 | 17 | // Shape of the tensor. 18 | TensorShapeProto shape = 2; 19 | 20 | // Information about the size and allocator used for the data 21 | AllocationDescription allocation_description = 4; 22 | }; 23 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/variable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VariableProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a Variable. 10 | message VariableDef { 11 | // Name of the variable tensor. 12 | string variable_name = 1; 13 | 14 | // Name of the initializer op. 15 | string initializer_name = 2; 16 | 17 | // Name of the snapshot tensor. 18 | string snapshot_name = 3; 19 | 20 | // Support for saving variables as slices of a larger variable. 21 | SaveSliceInfoDef save_slice_info_def = 4; 22 | } 23 | 24 | message SaveSliceInfoDef { 25 | // Name of the full variable of which this is a slice. 26 | string full_name = 1; 27 | // Shape of the full variable. 28 | repeated int32 full_shape = 2; 29 | // Offset of this variable into the full variable. 30 | repeated int32 var_offset = 3; 31 | // Shape of this variable. 32 | repeated int32 var_shape = 4; 33 | } 34 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/framework/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VersionsProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Version information for a piece of serialized data 10 | // 11 | // There are different types of versions for each type of data 12 | // (GraphDef, etc.), but they all have the same common shape 13 | // described here. 14 | // 15 | // Each consumer has "consumer" and "min_producer" versions (specified 16 | // elsewhere). A consumer is allowed to consume this data if 17 | // 18 | // producer >= min_producer 19 | // consumer >= min_consumer 20 | // consumer not in bad_consumers 21 | // 22 | message VersionDef { 23 | // The version of the code that produced this data. 24 | int32 producer = 1; 25 | 26 | // Any consumer below this version is not allowed to consume this data. 27 | int32 min_consumer = 2; 28 | 29 | // Specific consumer versions which are disallowed (e.g. due to bugs). 30 | repeated int32 bad_consumers = 3; 31 | }; 32 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/kernels/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ReaderBaseProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.kernels"; 8 | 9 | // For serializing and restoring the state of ReaderBase, see 10 | // reader_base.h for details. 11 | message ReaderBaseState { 12 | int64 work_started = 1; 13 | int64 work_finished = 2; 14 | int64 num_records_produced = 3; 15 | bytes current_work = 4; 16 | }; 17 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/protobuf/named_tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "NamedTensorProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/framework/tensor.proto"; 10 | 11 | // A pair of tensor name and tensor values. 12 | message NamedTensorProto { 13 | // Name of the tensor. 14 | string name = 1; 15 | 16 | // The client can populate a TensorProto using a tensorflow::Tensor`, or 17 | // directly using the protobuf field accessors. 18 | // 19 | // The client specifies whether the returned tensor values should be 20 | // filled tensor fields (float_val, int_val, etc.) or encoded in a 21 | // compact form in tensor.tensor_content. 22 | TensorProto tensor = 2; 23 | } 24 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/protobuf/queue_runner.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "QueueRunnerProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/lib/core/error_codes.proto"; 10 | 11 | // Protocol buffer representing a QueueRunner. 12 | message QueueRunnerDef { 13 | // Queue name. 14 | string queue_name = 1; 15 | 16 | // A list of enqueue operations. 17 | repeated string enqueue_op_name = 2; 18 | 19 | // The operation to run to close the queue. 20 | string close_op_name = 3; 21 | 22 | // The operation to run to cancel the queue. 23 | string cancel_op_name = 4; 24 | 25 | // A list of exception types considered to signal a safely closed queue 26 | // if raised during enqueue operations. 27 | repeated error.Code queue_closed_exception_types = 5; 28 | } 29 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/protobuf/saved_model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "SavedModelProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | import "tensorflow/core/protobuf/meta_graph.proto"; 10 | 11 | // SavedModel is the high level serialization format for TensorFlow Models. 12 | // See [todo: doc links, similar to session_bundle] for more information. 13 | message SavedModel { 14 | // The schema version of the SavedModel instance. Used for versioning when 15 | // making future changes to the specification/implementation. Initial value 16 | // at release will be 1. 17 | int64 saved_model_schema_version = 1; 18 | 19 | // One or more MetaGraphs. 20 | repeated MetaGraphDef meta_graphs = 2; 21 | } 22 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/core/util/memmapped_file_system.proto: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | syntax = "proto3"; 16 | 17 | package tensorflow; 18 | option cc_enable_arenas = true; 19 | 20 | // A message that describes one region of memmapped file. 21 | message MemmappedFileSystemDirectoryElement { 22 | uint64 offset = 1; 23 | string name = 2; 24 | } 25 | 26 | // A directory of regions in a memmapped file. 27 | message MemmappedFileSystemDirectory { 28 | repeated MemmappedFileSystemDirectoryElement element = 1; 29 | } 30 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/examples/udacity/.gitignore: -------------------------------------------------------------------------------- 1 | notMNIST_large* 2 | notMNIST_small* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/go/genop/.gitignore: -------------------------------------------------------------------------------- 1 | # .pb.go files generated by generate.sh 2 | internal/proto/* 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/go/op/.gitignore: -------------------------------------------------------------------------------- 1 | wrappers.go 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/python/framework/cpp_shape_inference.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow/core/framework/types.proto"; 7 | import "tensorflow/core/framework/tensor_shape.proto"; 8 | 9 | message CppShapeInferenceResult { 10 | TensorShapeProto shape = 1; 11 | TensorShapeProto handle_shape = 2; 12 | DataType handle_dtype = 3; 13 | } -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/python/training/checkpoint_state.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | 6 | // Protocol buffer representing the checkpoint state. 7 | // 8 | // TODO(touts): Add other attributes as needed. 9 | message CheckpointState { 10 | // Path to the most-recent model checkpoint. 11 | string model_checkpoint_path = 1; 12 | 13 | // Paths to all not-yet-deleted model checkpoints, sorted from oldest to 14 | // newest. 15 | // Note that the value of model_checkpoint_path should be the last item in 16 | // this list. 17 | repeated string all_model_checkpoint_paths = 2; 18 | } 19 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/python/util/protobuf/compare_test.proto: -------------------------------------------------------------------------------- 1 | // Test messages used in compare_test.py. 2 | syntax = "proto2"; 3 | 4 | package compare_test; 5 | option cc_enable_arenas = true; 6 | 7 | enum Enum { 8 | A = 0; 9 | B = 1; 10 | C = 2; 11 | } 12 | 13 | message Small { 14 | repeated string strings = 1; 15 | }; 16 | 17 | message Medium { 18 | repeated int32 int32s = 1; 19 | repeated Small smalls = 2; 20 | repeated group GroupA = 3 { 21 | repeated group GroupB = 4 { 22 | required string strings = 5; 23 | } 24 | } 25 | repeated float floats = 6; 26 | }; 27 | 28 | message Large { 29 | optional string string_ = 1; 30 | optional int64 int64_ = 2; 31 | optional float float_ = 3; 32 | optional bool bool_ = 4; 33 | optional Enum enum_ = 5; 34 | repeated int64 int64s = 6; 35 | optional Medium medium = 7; 36 | optional Small small = 8; 37 | optional double double_ = 9; 38 | optional WithMap with_map = 10; 39 | }; 40 | 41 | message Labeled { 42 | required int32 required = 1; 43 | optional int32 optional = 2; 44 | } 45 | 46 | message WithMap { 47 | map value_message = 1; 48 | map value_string = 2; 49 | } 50 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tensorboard/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory" : "components" 3 | } -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tensorboard/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | typings/* 3 | build/* 4 | dist/tf-tensorboard-demo.html 5 | 6 | # Since bower components are stored in the same directory as 7 | # tensorboard components, we ignore everything under components 8 | # except our own components which start with tf-. 9 | components/* 10 | # This rule should always be in sync with TF_COMPONENTS_TYPESCRIPT_GLOB 11 | # in gulpfile.js 12 | !components/tf-* 13 | !components/tf_* 14 | !components/vz-* 15 | !components/vz_* 16 | !components/index.html 17 | !components/BUILD 18 | # Ignore the sample graph files since they are too large to 19 | # be in the repo. 20 | components/tf-graph/demo/tf_model_zoo/* 21 | 22 | # All standalone code for TensorBoard components should be written in 23 | # typescript, and the compiled javascript code should be ignored. 24 | components/tf-*/**/*.js 25 | components/tf_*/**/*.js 26 | components/vz-*/**/*.js 27 | components/vz_*/**/*.js 28 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tools/ci_build/install/.bazelrc: -------------------------------------------------------------------------------- 1 | # Running bazel inside a `docker build` command causes trouble, cf: 2 | # https://github.com/bazelbuild/bazel/issues/134 3 | # The easiest solution is to set up a bazelrc file forcing --batch. 4 | startup --batch 5 | 6 | # Similarly, we need to workaround sandboxing issues: 7 | # https://github.com/bazelbuild/bazel/issues/418 8 | build --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone 9 | test --spawn_strategy=standalone 10 | 11 | # Force bazel output to use colors (good for jenkins) and print useful errors. 12 | common --color=yes 13 | 14 | # Configure tests - increase timeout, print errors and timeout warnings 15 | test --verbose_failures --test_output=errors --test_verbose_timeout_warnings 16 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tools/git/.gitignore: -------------------------------------------------------------------------------- 1 | gen 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tools/swig/.gitignore: -------------------------------------------------------------------------------- 1 | swig_path 2 | -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tools/tfprof/tfprof_log.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | message OpLogEntry { 6 | // op name. 7 | optional string name = 1; 8 | // float_ops is filled by tfprof Python API when called. It requires the 9 | // op has RegisterStatistics defined. Currently, Conv2D, MatMul, etc, are 10 | // implemented. 11 | optional int64 float_ops = 2; 12 | // User can define extra op type information for an op. This allows the user 13 | // to select a group of ops precisely using op_type as a key. 14 | repeated string types = 3; 15 | } 16 | 17 | message OpLog { 18 | repeated OpLogEntry log_entries = 1; 19 | } -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/generate_python_files/tensorflow/tools/tfprof/tfprof_options.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package tensorflow.tfprof; 4 | 5 | // Refers to tfprof_options.h/cc for documentation. 6 | // Only used to pass tfprof options from Python to C++. 7 | message OptionsProto { 8 | optional int64 max_depth = 1; 9 | optional int64 min_bytes = 2; 10 | optional int64 min_micros = 3; 11 | optional int64 min_params = 4; 12 | optional int64 min_float_ops = 5; 13 | repeated string device_regexes = 6; 14 | optional string order_by = 7; 15 | repeated string account_type_regexes = 8; 16 | repeated string start_name_regexes = 9; 17 | repeated string trim_name_regexes = 10; 18 | repeated string show_name_regexes = 11; 19 | repeated string hide_name_regexes = 12; 20 | optional bool account_displayed_op_only = 13; 21 | repeated string select = 14; 22 | optional bool viz = 15; 23 | optional string dump_to_file = 16; 24 | } -------------------------------------------------------------------------------- /tensorflow_template_application/python_predict_client/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio>=0.15.0 2 | grpcio-tools>=0.15.0 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow>=1.0.0 2 | scipy 3 | scikit-learn>=0.17.1 4 | coloredlogs>=5.2 5 | -------------------------------------------------------------------------------- /tensorflow_template_application/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | setuptools.setup(name='trainer', version='1.0', packages=['trainer']) 3 | -------------------------------------------------------------------------------- /tensorflow_template_application/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tensorflow_template_application/trainer/__init__.py -------------------------------------------------------------------------------- /tensorflow_transfer_cifar10/data_utils.py: -------------------------------------------------------------------------------- 1 | import cPickle as pickle 2 | import numpy as np 3 | import os 4 | 5 | # Make sure to download and extract CIFAR-10 data before 6 | # running this (https://www.cs.toronto.edu/~kriz/cifar.html) 7 | 8 | def load_CIFAR_batch(filename): 9 | """ load single batch of cifar """ 10 | with open(filename, 'rb') as f: 11 | datadict = pickle.load(f) 12 | X = datadict['data'] 13 | Y = datadict['labels'] 14 | X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") 15 | Y = np.array(Y) 16 | return X, Y 17 | 18 | def load_CIFAR10(ROOT): 19 | """ load all of cifar """ 20 | xs = [] 21 | ys = [] 22 | for b in range(1,6): 23 | f = os.path.join(ROOT, 'data_batch_%d' % (b, )) 24 | X, Y = load_CIFAR_batch(f) 25 | xs.append(X) 26 | ys.append(Y) 27 | Xtr = np.concatenate(xs) 28 | Ytr = np.concatenate(ys) 29 | del X, Y 30 | Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) 31 | return Xtr, Ytr, Xte, Yte 32 | -------------------------------------------------------------------------------- /tf-bpr/deploy_bpr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | # Define a timestamp function 5 | timestamp() { 6 | date +"%I_%M_%S" 7 | } 8 | 9 | 10 | export NAME=snado_amzn_bpr_$(timestamp)_$1 11 | echo $NAME 12 | 13 | 14 | gcloud beta ml jobs submit training $NAME \ 15 | --package-path=amzn/bpr \ 16 | --module-name=bpr.bpr \ 17 | --staging-bucket="gs://sharknado-team" \ 18 | --region=us-east1 \ 19 | --scale-tier=BASIC_GPU -------------------------------------------------------------------------------- /tf-bpr/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tf-bpr/models/__init__.py -------------------------------------------------------------------------------- /tf-bpr/models/hbpr.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | import numpy 4 | import random 5 | import time 6 | import numpy as np 7 | from corpus import Corpus 8 | from model import Model 9 | from vbpr import VBPR 10 | 11 | 12 | class HBPR(VBPR): 13 | def __init__(self, session, corpus, sampler, k, k2, factor_reg, bias_reg): 14 | #I have the image features in the corpus 15 | #now load the heuristic features: brand and price 16 | corpus.load_heuristics() 17 | #now just run VBPR: 18 | 19 | VBPR.__init__(self, session, corpus, sampler, k, k2, factor_reg, bias_reg) 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /tf-serving-demo/model/model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | from tensorflow.python.util import compat 4 | 5 | x = tf.placeholder(tf.float32, shape=(None)) 6 | y = tf.placeholder(tf.float32, shape=(None)) 7 | 8 | three = tf.Variable(3, dtype= tf.float32) 9 | z = tf.scalar_mul(three, x) + y 10 | 11 | model_version = 1 12 | path = os.path.join("three_x_plus_y", str(model_version)) 13 | builder = tf.python.saved_model.builder.SavedModelBuilder(path) 14 | 15 | with tf.Session() as sess: 16 | sess.run(tf.global_variables_initializer()) 17 | 18 | builder.add_meta_graph_and_variables( 19 | sess, 20 | [tf.python.saved_model.tag_constants.SERVING], 21 | signature_def_map= { 22 | "magic_model": tf.saved_model.signature_def_utils.predict_signature_def( 23 | inputs= {"egg": x, "bacon": y}, 24 | outputs= {"spam": z}) 25 | }) 26 | builder.save() 27 | -------------------------------------------------------------------------------- /tf-serving-demo/model/three_x_plus_y/1/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tf-serving-demo/model/three_x_plus_y/1/saved_model.pb -------------------------------------------------------------------------------- /tf-serving-demo/model/three_x_plus_y/1/variables/variables.data-00000-of-00001: -------------------------------------------------------------------------------- 1 | @@ -------------------------------------------------------------------------------- /tf-serving-demo/model/three_x_plus_y/1/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tf-serving-demo/model/three_x_plus_y/1/variables/variables.index -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow/core/framework/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ResourceHandleProto"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 10 | // not valid across executions, but can be serialized back and forth from within 11 | // a single run. 12 | message ResourceHandle { 13 | // Unique name for the device containing the resource. 14 | string device = 1; 15 | 16 | // Container in which this resource is placed. 17 | string container = 2; 18 | 19 | // Unique name of this resource. 20 | string name = 3; 21 | 22 | // Hash code for the type of the resource. Is only valid in the same device 23 | // and in the same execution. 24 | uint64 hash_code = 4; 25 | 26 | // For debug-only, the name of the type pointed to by this handle, if 27 | // available. 28 | string maybe_type_name = 5; 29 | }; 30 | -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow/core/framework/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "VersionsProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | 9 | // Version information for a piece of serialized data 10 | // 11 | // There are different types of versions for each type of data 12 | // (GraphDef, etc.), but they all have the same common shape 13 | // described here. 14 | // 15 | // Each consumer has "consumer" and "min_producer" versions (specified 16 | // elsewhere). A consumer is allowed to consume this data if 17 | // 18 | // producer >= min_producer 19 | // consumer >= min_consumer 20 | // consumer not in bad_consumers 21 | // 22 | message VersionDef { 23 | // The version of the code that produced this data. 24 | int32 producer = 1; 25 | 26 | // Any consumer below this version is not allowed to consume this data. 27 | int32 min_consumer = 2; 28 | 29 | // Specific consumer versions which are disallowed (e.g. due to bugs). 30 | repeated int32 bad_consumers = 3; 31 | }; 32 | -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow_serving/apis/get_model_metadata.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/any.proto"; 7 | import "tensorflow/core/protobuf/meta_graph.proto"; 8 | import "tensorflow_serving/apis/model.proto"; 9 | 10 | // Message returned for "signature_def" field. 11 | message SignatureDefMap { 12 | map signature_def = 1; 13 | }; 14 | 15 | message GetModelMetadataRequest { 16 | // Model Specification indicating which model we are querying for metadata. 17 | ModelSpec model_spec = 1; 18 | // Metadata fields to get. Currently supported: "signature_def". 19 | repeated string metadata_field = 2; 20 | } 21 | 22 | message GetModelMetadataResponse { 23 | // Model Specification indicating which model this metadata belongs to. 24 | ModelSpec model_spec = 1; 25 | // Map of metadata field name to metadata field. The options for metadata 26 | // field name are listed in GetModelMetadataRequest. Currently supported: 27 | // "signature_def". 28 | map metadata = 2; 29 | } 30 | -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow_serving/apis/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow_serving/apis/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow_serving/apis/classification.proto"; 7 | import "tensorflow_serving/apis/get_model_metadata.proto"; 8 | import "tensorflow_serving/apis/predict.proto"; 9 | import "tensorflow_serving/apis/regression.proto"; 10 | 11 | // open source marker; do not remove 12 | // PredictionService provides access to machine-learned models loaded by 13 | // model_servers. 14 | service PredictionService { 15 | // Classify. 16 | rpc Classify(ClassificationRequest) returns (ClassificationResponse); 17 | 18 | // Regress. 19 | rpc Regress(RegressionRequest) returns (RegressionResponse); 20 | 21 | // Predict -- provides access to loaded TensorFlow model. 22 | rpc Predict(PredictRequest) returns (PredictResponse); 23 | 24 | // GetModelMetadata - provides access to metadata for loaded models. 25 | rpc GetModelMetadata(GetModelMetadataRequest) 26 | returns (GetModelMetadataResponse); 27 | } 28 | -------------------------------------------------------------------------------- /tf-serving-demo/src/main/proto/tensorflow_serving/apis/regression.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option cc_enable_arenas = true; 4 | 5 | import "tensorflow_serving/apis/input.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | package tensorflow.serving; 9 | 10 | // Regression result for a single item 11 | // (tensorflow.Example or tensorflow.InferenceExample.features). 12 | message Regression { 13 | float value = 1; 14 | } 15 | 16 | // For tensorflow.Example this will contain one result. 17 | // For tensorflow.InferenceExample this will contain one result for each 18 | // InferenceExample::features. 19 | message RegressionResult { 20 | repeated Regression regressions = 1; 21 | } 22 | 23 | // RPC interfaces. 24 | 25 | message RegressionRequest { 26 | // Model Specification. 27 | ModelSpec model_spec = 1; 28 | 29 | // Input data. 30 | tensorflow.serving.Input input = 2; 31 | } 32 | 33 | message RegressionResponse { 34 | RegressionResult result = 1; 35 | } 36 | -------------------------------------------------------------------------------- /tf_deep_ctr/README.md: -------------------------------------------------------------------------------- 1 | # Practical TensorFlow Repository 2 | 3 | - deep_ctr： DeepFM/wide_n_deep/FNN/PNN/NFM/AFM/Deep & Cross Network等CTR预估相关深度学习算法 4 | - DeepRL： Deep-Reinforcement Learing 5 | - DeepMTL： Multi-task Learning 6 | - DeepTXT： DSSM等文本相关深度学习算法 7 | 8 | Welcome to pull requests! 9 | -------------------------------------------------------------------------------- /tf_deep_ctr/deep_ctr/Serving_pipeline/wide_n_deep_serving_client.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stasi009/DataScienceCodes/0ed2f4a8e2728917fc542363a9ca7700e94bbb4e/tf_deep_ctr/deep_ctr/Serving_pipeline/wide_n_deep_serving_client.h --------------------------------------------------------------------------------