├── xbert
├── layers
│ ├── __init__.py
│ ├── bias_add.py
│ ├── custom_decorator.py
│ ├── embedding_layer.py
│ ├── feed_forward.py
│ ├── position_embedding.py
│ ├── multi_head_attention.py
│ └── layer_normalization.py
├── __init__.py
├── utils.py
├── backend.py
├── tokenizer.py
└── xbert_model.py
├── testdemo
└── xbert_classification.py
├── .idea
├── dictionaries
│ └── xuyingjie.xml
├── thriftCompiler.xml
├── fileColors.xml
├── vcs.xml
├── modules.xml
├── misc.xml
└── xbert.iml
├── setup.py
├── README.md
└── LICENSE
/xbert/layers/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/testdemo/xbert_classification.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/xbert/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | __version__ = '0.1.0'
--------------------------------------------------------------------------------
/.idea/dictionaries/xuyingjie.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/thriftCompiler.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/fileColors.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/.idea/xbert.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from setuptools import setup, find_packages
4 |
5 | setup(
6 | name='xbert',
7 | version='0.1.0',
8 | description='universal xbert frame by tf2',
9 | long_description='xbert: https://github.com/xuyingjie521/xbert',
10 | license='GNU General Public License 3.0',
11 | url='https://github.com/xuyingjie521/xbert',
12 | author='yuyangmu',
13 | author_email='1812316597@163.com',
14 | install_requires=['tensorflow>=2.2.0'],
15 | packages=find_packages()
16 | )
17 |
--------------------------------------------------------------------------------
/xbert/layers/bias_add.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/24
8 |
9 | # the function of this file: Provide custom bias layer based on tf2
10 |
11 | from .custom_decorator import Layer, integerize_shape
12 | from xbert.backend import K
13 |
14 |
15 | class BiasAdd(Layer):
16 | """自定义bias层可辅助添加
17 | """
18 | @integerize_shape
19 | def build(self, input_shape):
20 | super(BiasAdd, self).build(input_shape)
21 | output_dim = input_shape[-1]
22 | self.bias = self.add_weight(
23 | name='bias',
24 | shape=(output_dim,),
25 | initializer='zeros',
26 | trainable=True
27 | )
28 |
29 | def call(self, inputs):
30 | return K.bias_add(inputs, self.bias)
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # xbert
2 | Implementation of pre-training model loading architecture of bert and its variants with tensorflow2
3 |
4 | ## Description
5 |
6 | This is based on the Transformer architecture implemented by tf2.keras, which can quickly load the pre-trained bert model for downstream finetune training.
7 | So welcome to star and I will continue to update in the future.
8 |
9 | ## Install
10 | Temporary support：
11 |
12 | ```bash
13 | pip install git+https://github.com/xuyingjie521/xbert.git
14 | ```
15 | ## Features
16 | Features that have been implemented so far：
17 |
18 | * Load pre-training weights of bert/roberta for finetune.
19 | * Support tf2.keras.
20 |
21 | ## Pre-trained models be loaded
22 |
23 | * Google original bert: https://github.com/google-research/bert
24 | * Harbin Institute of Technology version roberta: https://github.com/ymcui/Chinese-BERT-wwm
25 | * Brightmart version of roberta: https://github.com/brightmart/roberta_zh
26 |
--------------------------------------------------------------------------------
/xbert/utils.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/31
8 |
9 | # the function of this file: implement various auxiliary functions
10 |
11 |
12 | import sys
13 | import six
14 | import numpy as np
15 |
16 | is_py2 = six.PY2
17 |
18 | __all__ = ['is_sting', ]
19 |
20 |
21 | if not is_py2:
22 | basestring = str
23 |
24 |
25 | def is_sting(s):
26 | """
27 | 字符串判断函数
28 | :param s: 字符串
29 | :return: Boolean
30 | """
31 | return isinstance(s, basestring)
32 |
33 |
34 | def sequence_padding(inputs, length=None, padding=0):
35 | """Numpy函数，将序列padding到同一长度
36 | """
37 | if length is None:
38 | length = max([len(x) for x in inputs])
39 |
40 | pad_width = [(0, 0) for _ in np.shape(inputs[0])]
41 | outputs = []
42 | for x in inputs:
43 | x = x[:length]
44 | pad_width[0] = (0, length - len(x))
45 | x = np.pad(x, pad_width, 'constant', constant_values=padding)
46 | outputs.append(x)
47 |
48 | return np.array(outputs)
49 |
50 |
--------------------------------------------------------------------------------
/xbert/layers/custom_decorator.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/24
8 |
9 | # the function of this file: Provide custom decorator functions and Layer for tf.keras.layer based on tf2
10 |
11 |
12 | from xbert.backend import keras
13 |
14 |
15 | class Layer(keras.layers.Layer):
16 | """
17 | The custom layer of my project can be masked
18 | """
19 | def __init__(self, **kwargs):
20 | super(Layer, self).__init__(**kwargs)
21 | self.supports_masking = True
22 |
23 |
24 | def integerize_shape(func):
25 | """确保input_shape一定是int或None的自定义装饰器
26 | """
27 | def convert(item):
28 | if hasattr(item, '__iter__'):
29 | return [convert(i) for i in item]
30 | elif hasattr(item, 'value'):
31 | return item.value
32 | else:
33 | return item
34 |
35 | def new_func(self, input_shape):
36 | input_shape = convert(input_shape)
37 | return func(self, input_shape)
38 |
39 | return new_func
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/xbert/layers/embedding_layer.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/24
8 |
9 | # the function of this file: Provide custom embedding layer based on tf2
10 |
11 |
12 | from xbert.backend import keras, K
13 |
14 |
15 | class Embedding(keras.layers.Embedding):
16 | """Custom Embedding Layer
17 | """
18 | def compute_mask(self, inputs, mask=None):
19 | """first token is not masked
20 | """
21 | if self._current_mode == 'embedding':
22 | mask = super(Embedding, self).compute_mask(inputs, mask)
23 | if mask is not None:
24 | mask1 = K.ones_like(mask[:, :1], dtype='bool')
25 | mask2 = mask[:, 1:]
26 | return K.concatenate([mask1, mask2], 1)
27 | else:
28 | return mask
29 |
30 | def call(self, inputs, mode='embedding'):
31 | """define mode parameter:
32 | if 'embedding': Common Embedding layer
33 | elif 'dense' : Dense layer without bias
34 | """
35 | self._current_mode = mode
36 | if mode == 'embedding':
37 | return super(Embedding, self).call(inputs)
38 | else:
39 | kernel = K.transpose(self.embeddings)
40 | return K.dot(inputs, kernel)
41 |
42 | def compute_output_shape(self, input_shape):
43 | if self._current_mode == 'embedding':
44 | return super(Embedding, self).compute_output_shape(input_shape)
45 | else:
46 | return input_shape[:2] + (K.int_shape(self.embeddings)[0],)
47 |
--------------------------------------------------------------------------------
/xbert/layers/feed_forward.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/31
8 |
9 | # the function of this file: Provide custom feed forward layer based on tf2
10 |
11 |
12 | from tensorflow.keras import initializers, activations
13 | from .custom_decorator import Layer, integerize_shape
14 | from tensorflow.keras.layers import Dense
15 |
16 |
17 | class FeedForward(Layer):
18 | """
19 | 用两个dense层实现feed forward层
20 | """
21 | def __init__(
22 | self,
23 | units,
24 | activation='relu',
25 | use_bias=True,
26 | kernel_initializer='glorot_uniform',
27 | **kwargs
28 | ):
29 | super(FeedForward, self).__init__(**kwargs)
30 | self.units = units
31 | self.activation = activations.get(activation)
32 | self.use_bias = use_bias
33 | self.kernel_initializer = initializers.get(kernel_initializer)
34 |
35 | @integerize_shape
36 | def build(self, input_shape):
37 | super(FeedForward, self).build(input_shape)
38 | output_dim = input_shape[-1]
39 |
40 | self.dense_1 = Dense(units=self.units,
41 | activation=self.activation,
42 | use_bias=self.use_bias,
43 | kernel_initializer=self.kernel_initializer)
44 | self.dense_2 = Dense(units=output_dim,
45 | use_bias=self.use_bias,
46 | kernel_initializer=self.kernel_initializer)
47 |
48 | def call(self, inputs):
49 | x = inputs
50 | x = self.dense_1(x)
51 | x = self.dense_2(x)
52 | return x
53 |
54 | def get_config(self):
55 | config = {
56 | 'units': self.units,
57 | 'activation': activations.serialize(self.activation),
58 | 'use_bias': self.use_bias,
59 | 'kernel_initializer':
60 | initializers.serialize(self.kernel_initializer),
61 | }
62 | base_config = super(FeedForward, self).get_config()
63 | return dict(list(base_config.items()) + list(config.items()))
--------------------------------------------------------------------------------
/xbert/layers/position_embedding.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/31
8 |
9 | # the function of this file: Provide custom position embedding layer based on tf2
10 |
11 |
12 | from tensorflow.keras import initializers, activations
13 | from xbert.backend import K
14 | from .custom_decorator import Layer
15 | import tensorflow as tf
16 |
17 |
18 | class PositionEmbedding(Layer):
19 | """实现 Position Embedding，且Embedding可训练。
20 | """
21 | def __init__(
22 | self,
23 | input_dim,
24 | output_dim,
25 | merge_mode='add',
26 | embeddings_initializer='zeros',
27 | custom_position_ids=False,
28 | **kwargs
29 | ):
30 | super(PositionEmbedding, self).__init__(**kwargs)
31 | self.input_dim = input_dim
32 | self.output_dim = output_dim
33 | self.merge_mode = merge_mode
34 | self.embeddings_initializer = initializers.get(embeddings_initializer)
35 | self.custom_position_ids = custom_position_ids
36 |
37 | def build(self, input_shape):
38 | super(PositionEmbedding, self).build(input_shape)
39 | self.embeddings = self.add_weight(name='embeddings',
40 | shape=(self.input_dim, self.output_dim),
41 | initializer=self.embeddings_initializer
42 | )
43 |
44 | def call(self, inputs):
45 | """如果custom_position_ids，那么第二个输入为自定义的位置id
46 | """
47 | if self.custom_position_ids:
48 | inputs, position_ids = inputs
49 | if K.dtype(position_ids) != 'int32':
50 | position_ids = K.cast(position_ids, 'int32')
51 | pos_embeddings = K.gather(self.embeddings, position_ids)
52 | else:
53 | input_shape = K.shape(inputs)
54 | batch_size, seq_len = input_shape[0], input_shape[1]
55 | pos_embeddings = self.embeddings[:seq_len]
56 | pos_embeddings = K.expand_dims(pos_embeddings, 0)
57 | if self.merge_mode != 'add':
58 | pos_embeddings = K.tile(pos_embeddings, [batch_size, 1, 1])
59 |
60 | if self.merge_mode == 'add':
61 | return inputs + pos_embeddings
62 | else:
63 | return K.concatenate([inputs, pos_embeddings])
64 |
65 | def compute_output_shape(self, input_shape):
66 | if self.custom_position_ids:
67 | input_shape = input_shape[0]
68 |
69 | if self.merge_mode == 'add':
70 | return input_shape
71 | else:
72 | return input_shape[:2] + (input_shape[2] + self.output_dim,)
73 |
74 | def get_config(self):
75 | config = {
76 | 'input_dim': self.input_dim,
77 | 'output_dim': self.output_dim,
78 | 'merge_mode': self.merge_mode,
79 | 'embeddings_initializer':
80 | initializers.serialize(self.embeddings_initializer),
81 | 'custom_position_ids': self.custom_position_ids,
82 | }
83 | base_config = super(PositionEmbedding, self).get_config()
84 | return dict(list(base_config.items()) + list(config.items()))
85 |
86 |
87 | class SinCosPositionEmbedding(Layer):
88 | """
89 | 实现 Google 论文中的 Sin-Cos 形式的位置 Embedding 层
90 | """
91 | def __init__(self, v_dim, merge_mode="add", **kwargs):
92 | """
93 | Args:
94 | v_dim: embedding 的维度
95 | merge_mode: 与位置 embedding 的合并模式， "add" 表示相加，"concate" 表示拼接
96 | **kwargs:
97 | """
98 | super(SinCosPositionEmbedding, self).__init__(**kwargs)
99 | self.v_dim = v_dim
100 | self.merge_mode = merge_mode
101 |
102 | def call(self, inputs):
103 | pid = tf.range(K.shape(inputs)[1])
104 | pid = K.expand_dims(pid, 0)
105 | pid = K.tile(pid, [K.shape(inputs)[0], 1])
106 | pv = self.idx2pos(pid)
107 | if self.merge_mode == "add":
108 | return pv + inputs
109 | else:
110 | return K.concatenate([inputs, pv])
111 |
112 | def idx2pos(self, pid):
113 | pid = K.cast(pid, "float32")
114 | pid = K.expand_dims(pid, 2)
115 | pj = 1. / K.pow(10000., 2. / self.v_dim * K.arange(self.v_dim // 2,
116 | dtype="float32"))
117 | pj = K.expand_dims(pj, 0)
118 | pv = K.dot(pid, pj)
119 | pv1, pv2 = K.sin(pv), K.cos(pv)
120 | pv1, pv2 = K.expand_dims(pv1, 3), K.expand_dims(pv2, 3)
121 | pv = K.concatenate([pv1, pv2], 3)
122 | return K.reshape(pv, (K.shape(pv)[0], K.shape(pv)[1], self.v_dim))
123 |
124 | def compute_output_shape(self, input_shape):
125 | if self.merge_mode == "add":
126 | return input_shape
127 | else:
128 | return input_shape[:-1] + (input_shape[-1] + self.vdim)
129 |
130 | def get_config(self):
131 | config = {"v_dim": self.v_dim, "merge_mode": self.merge_mode}
132 | base_config = super(SinCosPositionEmbedding, self).get_config()
133 | return dict(list(base_config.items()) + list(config.items()))
134 |
--------------------------------------------------------------------------------
/xbert/layers/multi_head_attention.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/24
8 |
9 | # the function of this file: Provide custom Multiple Head Attention layer based on tf2
10 |
11 | import tensorflow as tf
12 | from tensorflow.keras import initializers, activations
13 | from tensorflow.keras.layers import *
14 | from xbert.backend import K, sequence_masking
15 | from .custom_decorator import Layer
16 |
17 |
18 | class MultiHeadAttention(Layer):
19 | """实现多头注意力层
20 | """
21 | def __init__(
22 | self,
23 | num_heads,
24 | head_size,
25 | key_size=None,
26 | use_bias=True,
27 | attention_scale=True,
28 | kernel_initializer='glorot_uniform',
29 | **kwargs
30 | ):
31 | """
32 | Args:
33 | num_heads: 多头 attention 的数量
34 | head_size: 每个 attention head 的维度
35 | key_size: 输出的 Embedding 维度
36 | **kwargs:
37 | """
38 | super(MultiHeadAttention, self).__init__(**kwargs)
39 | self.heads = num_heads
40 | self.head_size = head_size
41 | self.out_dim = num_heads * head_size
42 | self.key_size = key_size or head_size
43 | self.use_bias = use_bias
44 | self.attention_scale = attention_scale
45 | self.kernel_initializer = initializers.get(kernel_initializer)
46 |
47 | def build(self, input_shape):
48 | super(MultiHeadAttention, self).build(input_shape)
49 | self.q_dense = Dense(
50 | units=self.key_size * self.heads,
51 | use_bias=self.use_bias,
52 | kernel_initializer=self.kernel_initializer
53 | )
54 | self.k_dense = Dense(
55 | units=self.key_size * self.heads,
56 | use_bias=self.use_bias,
57 | kernel_initializer=self.kernel_initializer
58 | )
59 | self.v_dense = Dense(
60 | units=self.out_dim,
61 | use_bias=self.use_bias,
62 | kernel_initializer=self.kernel_initializer
63 | )
64 | self.o_dense = Dense(
65 | units=self.out_dim,
66 | use_bias=self.use_bias,
67 | kernel_initializer=self.kernel_initializer
68 | )
69 |
70 | def call(self, inputs, mask=None, a_mask=None, p_bias=None):
71 | """
72 | 实现多头注意力机制
73 | Args:
74 | inputs: 输入是一个 list，[q, k, v, mask, a_mask, p_bias]，mask 可少，q,k,v 不可少。
75 | q_mask: 对输入的query序列的mask，主要是将输出结果的padding部分置0。
76 | v_mask: 对输入的value序列的mask，主要是防止attention读取到padding信息。
77 | a_mask: 对attention矩阵的mask，不同的attention mask对应不同的应用。
78 | p_bias: 在attention里的位置偏置。一般用来指定相对位置编码的种类。
79 | **kwargs:
80 | Returns:返回经过注意力机制的结果
81 | """
82 | q, k, v = inputs[:3]
83 | q_mask, v_mask, n = None, None, 3
84 | if mask is not None:
85 | if mask[0] is not None:
86 | q_mask = K.cast(mask[0], K.floatx())
87 | if mask[2] is not None:
88 | v_mask = K.cast(mask[2], K.floatx())
89 | if a_mask:
90 | a_mask = inputs[n]
91 | n += 1
92 | # 线性变换
93 | qw = self.q_dense(q) # [batch_size, seq_len, num_heads * key_size]
94 | kw = self.k_dense(k) # [batch_size, seq_len, num_heads * key_size]
95 | vw = self.v_dense(v) # [batch_size, seq_len, num_heads * head_size]
96 |
97 | # 形状变换
98 | # [batch_size, seq_len, num_heads, key_size]
99 | qw = K.reshape(qw, (-1, K.shape(q)[1], self.heads, self.key_size))
100 | # [batch_size, seq_len, num_heads, key_size]
101 | kw = K.reshape(kw, (-1, K.shape(k)[1], self.heads, self.key_size))
102 | # [batch_size, seq_len, num_heads, head_size]
103 | vw = K.reshape(vw, (-1, K.shape(v)[1], self.heads, self.head_size))
104 |
105 | # Attention
106 | a = tf.einsum('bjhd,bkhd->bhjk', qw, kw)
107 | # 处理位置编码
108 | if p_bias == 'typical_relative':
109 | pos_embeddings = inputs[n]
110 | a = a + tf.einsum('bjhd,jkd->bhjk', qw, pos_embeddings)
111 | # Attention（续）
112 | if self.attention_scale:
113 | a = a / self.key_size**0.5
114 | a = sequence_masking(a, v_mask, 1, -1)
115 | if a_mask is not None:
116 | a = a - (1 - a_mask) * 1e12
117 | a = K.softmax(a)
118 |
119 | # 完成输出
120 | o = tf.einsum('bhjk,bkhd->bjhd', a, vw)
121 | if p_bias == 'typical_relative':
122 | o = o + tf.einsum('bhjk,jkd->bjhd', a, pos_embeddings)
123 | o = K.reshape(o, (-1, K.shape(o)[1], self.out_dim))
124 | o = self.o_dense(o)
125 | # 返回结果
126 | o = sequence_masking(o, q_mask, 0)
127 | return o
128 |
129 | def compute_output_shape(self, input_shape):
130 | return (input_shape[0][0], input_shape[0][1], self.out_dim)
131 |
132 | def compute_mask(self, inputs, mask=None):
133 | if mask is not None:
134 | return mask[0]
135 |
136 | def get_config(self):
137 | config = {
138 | 'heads': self.heads,
139 | 'head_size': self.head_size,
140 | 'key_size': self.key_size,
141 | 'use_bias': self.use_bias,
142 | 'attention_scale': self.attention_scale,
143 | 'kernel_initializer':
144 | initializers.serialize(self.kernel_initializer),
145 | }
146 | base_config = super(MultiHeadAttention, self).get_config()
147 | return dict(list(base_config.items()) + list(config.items()))
148 |
--------------------------------------------------------------------------------
/xbert/backend.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/22
8 |
9 | # the function of this file: Provide custom backend functions based on tf2
10 |
11 |
12 | import sys
13 | import numpy as np
14 | import tensorflow as tf
15 |
16 | from tensorflow.python.ops.custom_gradient import _graph_mode_decorator
17 | import tensorflow.keras as keras
18 | import tensorflow.keras.backend as K
19 | # 默认都是直接基于tf2
20 | sys.modules['keras'] = keras
21 | sys.modules['K'] = K
22 |
23 |
24 | def gelu_erf(x):
25 | """基于Erf的直接计算
26 | """
27 | return 0.5 * x * (1.0 + tf.math.erf(x / np.sqrt(2.0)))
28 |
29 |
30 | def gelu_tanh(x):
31 | """基于Tanh近似的计算
32 | """
33 | cdf = 0.5 * (
34 | 1.0 + K.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * K.pow(x, 3))))
35 | )
36 | return x * cdf
37 |
38 |
39 | def set_gelu(version):
40 | """设置gelu版本
41 | """
42 | version = version.lower()
43 | assert version in ['erf', 'tanh'], 'gelu version must be erf or tanh'
44 | if version == 'erf':
45 | keras.utils.get_custom_objects()['gelu'] = gelu_erf
46 | else:
47 | keras.utils.get_custom_objects()['gelu'] = gelu_tanh
48 |
49 |
50 | def piecewise_linear(t, schedule):
51 | """分段线性函数
52 | 其中schedule是形如{1000: 1, 2000: 0.1}的字典，
53 | 表示 t ∈ [0, 1000]时，输出从0均匀增加至1，而
54 | t ∈ [1000, 2000]时，输出从1均匀降低到0.1，最后
55 | t > 2000时，保持0.1不变。
56 | """
57 | schedule = sorted(schedule.items())
58 | if schedule[0][0] != 0:
59 | schedule = [(0, 0.0)] + schedule
60 |
61 | x = K.constant(schedule[0][1], dtype=K.floatx())
62 | t = K.cast(t, K.floatx())
63 | for i in range(len(schedule)):
64 | t_begin = schedule[i][0]
65 | x_begin = x
66 | if i != len(schedule) - 1:
67 | dx = schedule[i + 1][1] - schedule[i][1]
68 | dt = schedule[i + 1][0] - schedule[i][0]
69 | slope = 1.0 * dx / dt
70 | x = schedule[i][1] + slope * (t - t_begin)
71 | else:
72 | x = K.constant(schedule[i][1], dtype=K.floatx())
73 | x = K.switch(t >= t_begin, x, x_begin)
74 |
75 | return x
76 |
77 |
78 | def search_layer(inputs, name, exclude_from=None):
79 | """根据inputs和name来搜索层
80 | 说明：inputs为某个层或某个层的输出；name为目标层的名字。
81 | 实现：根据inputs一直往上递归搜索，直到发现名字为name的层为止；
82 | 如果找不到，那就返回None。
83 | """
84 | if exclude_from is None:
85 | exclude_from = set()
86 |
87 | if isinstance(inputs, keras.layers.Layer):
88 | layer = inputs
89 | else:
90 | layer = inputs._keras_history[0]
91 |
92 | if layer.name == name:
93 | return layer
94 | elif layer in exclude_from:
95 | return None
96 | else:
97 | exclude_from.add(layer)
98 | if isinstance(layer, keras.models.Model):
99 | model = layer
100 | for layer in model.layers:
101 | if layer.name == name:
102 | return layer
103 | inbound_layers = layer._inbound_nodes[0].inbound_layers
104 | if not isinstance(inbound_layers, list):
105 | inbound_layers = [inbound_layers]
106 | if len(inbound_layers) > 0:
107 | for layer in inbound_layers:
108 | layer = search_layer(layer, name, exclude_from)
109 | if layer is not None:
110 | return layer
111 |
112 |
113 | def sequence_masking(x, mask, mode=0, axis=None):
114 | """为序列条件mask的函数
115 | mask: 形如(batch_size, seq_len)的0-1矩阵；
116 | mode: 如果是0，则直接乘以mask；
117 | 如果是1，则在padding部分减去一个大正数。
118 | axis: 序列所在轴，默认为1；
119 | """
120 | if mask is None or mode not in [0, 1]:
121 | return x
122 | else:
123 | if axis is None:
124 | axis = 1
125 | if axis == -1:
126 | axis = K.ndim(x) - 1
127 | assert axis > 0, 'axis muse be greater than 0'
128 | for _ in range(axis - 1):
129 | mask = K.expand_dims(mask, 1)
130 | for _ in range(K.ndim(x) - K.ndim(mask) - axis + 1):
131 | mask = K.expand_dims(mask, K.ndim(mask))
132 | if mode == 0:
133 | return x * mask
134 | else:
135 | return x - (1 - mask) * 1e12
136 |
137 |
138 | def batch_gather(params, indices):
139 | """同tf旧版本的batch_gather
140 | """
141 | try:
142 | return tf.gather(params, indices, batch_dims=-1)
143 | except Exception as e1:
144 | try:
145 | return tf.batch_gather(params, indices)
146 | except Exception as e2:
147 | raise ValueError('%s\n%s\n' % (e1.message, e2.message))
148 |
149 |
150 | def divisible_temporal_padding(x, n):
151 | """将一维向量序列右padding到长度能被n整除
152 | """
153 | r_len = K.shape(x)[1] % n
154 | p_len = K.switch(r_len > 0, n - r_len, 0)
155 | return K.temporal_padding(x, (0, p_len))
156 |
157 |
158 | def swish(x):
159 | """swish函数（这样封装过后才有 __name__ 属性）
160 | """
161 | return tf.nn.swish(x)
162 |
163 |
164 | def leaky_relu(x, alpha=0.2):
165 | """leaky relu函数（这样封装过后才有 __name__ 属性）
166 | """
167 | return tf.nn.leaky_relu(x, alpha=alpha)
168 |
169 |
170 | def graph_mode_decorator(f, *args, **kwargs):
171 | """tf2.1与之前版本的传参方式不一样，这里做个同步
172 | """
173 | if tf.__version__ < '2.1':
174 | return _graph_mode_decorator(f, *args, **kwargs)
175 | else:
176 | return _graph_mode_decorator(f, args, kwargs)
177 |
178 |
179 | custom_objects = {
180 | 'gelu_erf': gelu_erf,
181 | 'gelu_tanh': gelu_tanh,
182 | 'gelu': gelu_erf,
183 | 'swish': swish,
184 | 'leaky_relu': leaky_relu,
185 | }
186 |
187 | keras.utils.get_custom_objects().update(custom_objects)
188 |
189 |
--------------------------------------------------------------------------------
/xbert/layers/layer_normalization.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/31
8 |
9 | # the function of this file: Provide custom normalization layer based on tf2
10 |
11 |
12 | from tensorflow.keras import initializers, activations
13 | from tensorflow.keras.layers import Dense
14 | from xbert.backend import K
15 | from .custom_decorator import Layer
16 |
17 |
18 | class BasicLayerNormalization(Layer):
19 | """
20 | 实现Basic Layer Normalization 层
21 | """
22 | def __init__(self, **kwargs):
23 | super(BasicLayerNormalization, self).__init__(**kwargs)
24 | self.epsilon = K.epsilon() * K.epsilon() # 初始化一个很小的数 1e-14
25 |
26 | def build(self, input_shape):
27 | super(BasicLayerNormalization, self).build(input_shape)
28 | shape = (input_shape[-1],)
29 | self.gamma = self.add_weight(shape=shape,
30 | initializer="ones",
31 | name="gamma")
32 | self.beta = self.add_weight(shape=shape,
33 | initializer="zeros",
34 | name="beta")
35 |
36 | def call(self, inputs):
37 | mean = K.mean(inputs, axis=-1, keepdims=True)
38 | variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
39 | std = K.sqrt(variance + self.epsilon) # 增加一个很小的数避免开根号求导梯度出现问题
40 | outputs = (inputs - mean) / std
41 | outputs *= self.gamma
42 | outputs += self.beta
43 | return outputs
44 |
45 | def get_config(self): # 增加 get_config 方法使得 tf.keras 模型能够保存为 h5
46 | config = {"epsilon": self.epsilon}
47 | base_config = super(BasicLayerNormalization, self).get_config()
48 | return dict(list(base_config.items()) + list(config.items()))
49 |
50 |
51 | class ConditionalLayerNormalization(Layer):
52 | """
53 | 实现Conditional Layer Normalization 层
54 | 当参数hidden_*仅为有输入条件时使用，即conditional=True
55 | """
56 | def __init__(
57 | self,
58 | center=True,
59 | scale=True,
60 | epsilon=None,
61 | conditional=False,
62 | hidden_units=None,
63 | hidden_activation='linear',
64 | hidden_initializer='glorot_uniform',
65 | **kwargs
66 | ):
67 | super(ConditionalLayerNormalization, self).__init__(**kwargs)
68 | self.center = center
69 | self.scale = scale
70 | self.conditional = conditional
71 | self.hidden_units = hidden_units
72 | self.hidden_activation = activations.get(hidden_activation)
73 | self.hidden_initializer = initializers.get(hidden_initializer)
74 | self.epsilon = epsilon or 1e-12
75 |
76 | def compute_mask(self, inputs, mask=None):
77 | if self.conditional:
78 | masks = [K.expand_dims(m, 0) for m in mask if m is not None]
79 | if len(masks) == 0:
80 | return None
81 | else:
82 | return K.all(K.concatenate(masks, axis=0), axis=0)
83 | else:
84 | return mask
85 |
86 | def build(self, input_shape):
87 | super(ConditionalLayerNormalization, self).build(input_shape)
88 |
89 | if self.conditional:
90 | shape = (input_shape[0][-1],)
91 | else:
92 | shape = (input_shape[-1],)
93 |
94 | if self.center:
95 | self.beta = self.add_weight(shape=shape,
96 | initializer='zeros',
97 | name='beta')
98 | if self.scale:
99 | self.gamma = self.add_weight(shape=shape,
100 | initializer='ones',
101 | name='gamma')
102 |
103 | if self.conditional:
104 |
105 | if self.hidden_units is not None:
106 | self.hidden_dense = Dense(
107 | units=self.hidden_units,
108 | activation=self.hidden_activation,
109 | use_bias=False,
110 | kernel_initializer=self.hidden_initializer)
111 |
112 | if self.center:
113 | self.beta_dense = Dense(units=shape[0],
114 | use_bias=False,
115 | kernel_initializer='zeros')
116 | if self.scale:
117 | self.gamma_dense = Dense(units=shape[0],
118 | use_bias=False,
119 | kernel_initializer='zeros')
120 |
121 | def call(self, inputs):
122 | """
123 | 如果是条件Layer Norm，则默认以list为输入，第二个是condition
124 | """
125 | if self.conditional:
126 | inputs, cond = inputs
127 | if self.hidden_units is not None:
128 | cond = self.hidden_dense(cond)
129 | for _ in range(K.ndim(inputs) - K.ndim(cond)):
130 | cond = K.expand_dims(cond, 1)
131 | if self.center:
132 | beta = self.beta_dense(cond) + self.beta
133 | if self.scale:
134 | gamma = self.gamma_dense(cond) + self.gamma
135 | else:
136 | if self.center:
137 | beta = self.beta
138 | if self.scale:
139 | gamma = self.gamma
140 |
141 | outputs = inputs
142 | if self.center:
143 | mean = K.mean(outputs, axis=-1, keepdims=True)
144 | outputs = outputs - mean
145 | if self.scale:
146 | variance = K.mean(K.square(outputs), axis=-1, keepdims=True)
147 | std = K.sqrt(variance + self.epsilon)
148 | outputs = outputs / std
149 | outputs = outputs * gamma
150 | if self.center:
151 | outputs = outputs + beta
152 |
153 | return outputs
154 |
155 | def compute_output_shape(self, input_shape):
156 | if self.conditional:
157 | return input_shape[0]
158 | else:
159 | return input_shape
160 |
161 | def get_config(self):
162 | config = {
163 | 'center': self.center,
164 | 'scale': self.scale,
165 | 'epsilon': self.epsilon,
166 | 'conditional': self.conditional,
167 | 'hidden_units': self.hidden_units,
168 | 'hidden_activation': activations.serialize(self.hidden_activation),
169 | 'hidden_initializer':
170 | initializers.serialize(self.hidden_initializer),
171 | }
172 | base_config = super(ConditionalLayerNormalization, self).get_config()
173 | return dict(list(base_config.items()) + list(config.items()))
174 |
175 |
--------------------------------------------------------------------------------
/xbert/tokenizer.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: CyberZHG
7 | # @date: 2019/09/01
8 |
9 | # the function of this file: implement Tokenizer
10 |
11 |
12 | import unicodedata
13 |
14 |
15 | TOKEN_PAD = '' # Token for padding
16 | TOKEN_UNK = '[UNK]' # Token for unknown words
17 | TOKEN_CLS = '[CLS]' # Token for classification
18 | TOKEN_SEP = '[SEP]' # Token for separation
19 | TOKEN_MASK = '[MASK]' # Token for masking
20 |
21 |
22 | class Tokenizer(object):
23 |
24 | def __init__(self,
25 | token_dict,
26 | token_cls=TOKEN_CLS,
27 | token_sep=TOKEN_SEP,
28 | token_unk=TOKEN_UNK,
29 | pad_index=0,
30 | cased=False):
31 | """Initialize tokenizer.
32 | :param token_dict: A dict maps tokens to indices.
33 | :param token_cls: The token represents classification.
34 | :param token_sep: The token represents separator.
35 | :param token_unk: The token represents unknown token.
36 | :param pad_index: The index to pad.
37 | :param cased: Whether to keep the case.
38 | """
39 | self._token_dict = token_dict
40 | self._token_dict_inv = {v: k for k, v in token_dict.items()}
41 | self._token_cls = token_cls
42 | self._token_sep = token_sep
43 | self._token_unk = token_unk
44 | self._pad_index = pad_index
45 | self._cased = cased
46 |
47 | @staticmethod
48 | def _truncate(first_tokens, second_tokens=None, max_len=None):
49 | if max_len is None:
50 | return
51 |
52 | if second_tokens is not None:
53 | while True:
54 | total_len = len(first_tokens) + len(second_tokens)
55 | if total_len <= max_len - 3: # 3 for [CLS] .. tokens_a .. [SEP] .. tokens_b [SEP]
56 | break
57 | if len(first_tokens) > len(second_tokens):
58 | first_tokens.pop()
59 | else:
60 | second_tokens.pop()
61 | else:
62 | del first_tokens[max_len - 2:] # 2 for [CLS] .. tokens .. [SEP]
63 |
64 | def _pack(self, first_tokens, second_tokens=None):
65 | first_packed_tokens = [self._token_cls] + first_tokens + [self._token_sep]
66 | if second_tokens is not None:
67 | second_packed_tokens = second_tokens + [self._token_sep]
68 | return first_packed_tokens + second_packed_tokens, len(first_packed_tokens), len(second_packed_tokens)
69 | else:
70 | return first_packed_tokens, len(first_packed_tokens), 0
71 |
72 | def _convert_tokens_to_ids(self, tokens):
73 | unk_id = self._token_dict.get(self._token_unk)
74 | return [self._token_dict.get(token, unk_id) for token in tokens]
75 |
76 | def tokenize(self, first, second=None):
77 | """Split text to tokens.
78 | :param first: First text.
79 | :param second: Second text.
80 | :return: A list of strings.
81 | """
82 | first_tokens = self._tokenize(first)
83 | second_tokens = self._tokenize(second) if second is not None else None
84 | tokens, _, _ = self._pack(first_tokens, second_tokens)
85 | return tokens
86 |
87 | def encode(self, first, second=None, max_len=None):
88 | first_tokens = self._tokenize(first)
89 | second_tokens = self._tokenize(second) if second is not None else None
90 | self._truncate(first_tokens, second_tokens, max_len)
91 | tokens, first_len, second_len = self._pack(first_tokens, second_tokens)
92 |
93 | token_ids = self._convert_tokens_to_ids(tokens)
94 | segment_ids = [0] * first_len + [1] * second_len
95 |
96 | if max_len is not None:
97 | pad_len = max_len - first_len - second_len
98 | token_ids += [self._pad_index] * pad_len
99 | segment_ids += [0] * pad_len
100 |
101 | return token_ids, segment_ids
102 |
103 | def decode(self, ids):
104 | sep = ids.index(self._token_dict[self._token_sep])
105 | try:
106 | stop = ids.index(self._pad_index)
107 | except ValueError as e:
108 | stop = len(ids)
109 | tokens = [self._token_dict_inv[i] for i in ids]
110 | first = tokens[1:sep]
111 | if sep < stop - 1:
112 | second = tokens[sep + 1:stop - 1]
113 | return first, second
114 | return first
115 |
116 | def _tokenize(self, text):
117 | if not self._cased:
118 | text = unicodedata.normalize('NFD', text)
119 | text = ''.join([ch for ch in text if unicodedata.category(ch) != 'Mn'])
120 | text = text.lower()
121 | spaced = ''
122 | for ch in text:
123 | if self._is_punctuation(ch) or self._is_cjk_character(ch):
124 | spaced += ' ' + ch + ' '
125 | elif self._is_space(ch):
126 | spaced += ' '
127 | elif ord(ch) == 0 or ord(ch) == 0xfffd or self._is_control(ch):
128 | continue
129 | else:
130 | spaced += ch
131 | tokens = []
132 | for word in spaced.strip().split():
133 | tokens += self._word_piece_tokenize(word)
134 | return tokens
135 |
136 | def _word_piece_tokenize(self, word):
137 | if word in self._token_dict:
138 | return [word]
139 | tokens = []
140 | start, stop = 0, 0
141 | while start < len(word):
142 | stop = len(word)
143 | while stop > start:
144 | sub = word[start:stop]
145 | if start > 0:
146 | sub = '##' + sub
147 | if sub in self._token_dict:
148 | break
149 | stop -= 1
150 | if start == stop:
151 | stop += 1
152 | tokens.append(sub)
153 | start = stop
154 | return tokens
155 |
156 | @staticmethod
157 | def _is_punctuation(ch):
158 | code = ord(ch)
159 | return 33 <= code <= 47 or \
160 | 58 <= code <= 64 or \
161 | 91 <= code <= 96 or \
162 | 123 <= code <= 126 or \
163 | unicodedata.category(ch).startswith('P')
164 |
165 | @staticmethod
166 | def _is_cjk_character(ch):
167 | code = ord(ch)
168 | return 0x4E00 <= code <= 0x9FFF or \
169 | 0x3400 <= code <= 0x4DBF or \
170 | 0x20000 <= code <= 0x2A6DF or \
171 | 0x2A700 <= code <= 0x2B73F or \
172 | 0x2B740 <= code <= 0x2B81F or \
173 | 0x2B820 <= code <= 0x2CEAF or \
174 | 0xF900 <= code <= 0xFAFF or \
175 | 0x2F800 <= code <= 0x2FA1F
176 |
177 | @staticmethod
178 | def _is_space(ch):
179 | return ch == ' ' or ch == '\n' or ch == '\r' or ch == '\t' or \
180 | unicodedata.category(ch) == 'Zs'
181 |
182 | @staticmethod
183 | def _is_control(ch):
184 | return unicodedata.category(ch) in ('Cc', 'Cf')
185 |
186 | @staticmethod
187 | def rematch(text, tokens, cased=False, unknown_token=TOKEN_UNK):
188 | """Try to find the indices of tokens in the original text.
189 | >>> Tokenizer.rematch("All rights reserved.", ["all", "rights", "re", "##ser", "##ved", "."])
190 | [(0, 3), (4, 10), (11, 13), (13, 16), (16, 19), (19, 20)]
191 | >>> Tokenizer.rematch("All rights reserved.", ["all", "rights", "re", "##ser", "[UNK]", "."])
192 | [(0, 3), (4, 10), (11, 13), (13, 16), (16, 19), (19, 20)]
193 | >>> Tokenizer.rematch("All rights reserved.", ["[UNK]", "rights", "[UNK]", "##ser", "[UNK]", "[UNK]"])
194 | [(0, 3), (4, 10), (11, 13), (13, 16), (16, 19), (19, 20)]
195 | >>> Tokenizer.rematch("All rights reserved.", ["[UNK]", "righs", "[UNK]", "ser", "[UNK]", "[UNK]"])
196 | [(0, 3), (4, 10), (11, 13), (13, 16), (16, 19), (19, 20)]
197 | >>> Tokenizer.rematch("All rights reserved.",
198 | ... ["[UNK]", "rights", "[UNK]", "[UNK]", "[UNK]", "[UNK]"]) # doctest:+ELLIPSIS
199 | [(0, 3), (4, 10), (11, ... 19), (19, 20)]
200 | >>> Tokenizer.rematch("All rights reserved.", ["all rights", "reserved", "."])
201 | [(0, 10), (11, 19), (19, 20)]
202 | >>> Tokenizer.rematch("All rights reserved.", ["all rights", "reserved", "."], cased=True)
203 | [(0, 10), (11, 19), (19, 20)]
204 | >>> Tokenizer.rematch("#hash tag ##", ["#", "hash", "tag", "##"])
205 | [(0, 1), (1, 5), (6, 9), (10, 12)]
206 | >>> Tokenizer.rematch("嘛呢，吃了吗？", ["[UNK]", "呢", "，", "[UNK]", "了", "吗", "？"])
207 | [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7)]
208 | >>> Tokenizer.rematch(" 吃了吗？ ", ["吃", "了", "吗", "？"])
209 | [(2, 3), (3, 4), (4, 5), (5, 6)]
210 | :param text: Original text.
211 | :param tokens: Decoded list of tokens.
212 | :param cased: Whether it is cased.
213 | :param unknown_token: The representation of unknown token.
214 | :return: A list of tuples represents the start and stop locations in the original text.
215 | """
216 | decoded, token_offsets = '', []
217 | for token in tokens:
218 | token_offsets.append([len(decoded), 0])
219 | if token == unknown_token:
220 | token = '#'
221 | if not cased:
222 | token = token.lower()
223 | if len(token) > 2 and token.startswith('##'):
224 | token = token[2:]
225 | elif len(decoded) > 0:
226 | token = ' ' + token
227 | token_offsets[-1][0] += 1
228 | decoded += token
229 | token_offsets[-1][1] = len(decoded)
230 |
231 | heading = 0
232 | text = text.rstrip()
233 | for i in range(len(text)):
234 | if not Tokenizer._is_space(text[i]):
235 | break
236 | heading += 1
237 | text = text[heading:]
238 | len_text, len_decode = len(text), len(decoded)
239 | costs = [[0] * (len_decode + 1) for _ in range(2)]
240 | paths = [[(-1, -1)] * (len_decode + 1) for _ in range(len_text + 1)]
241 | curr, prev = 0, 1
242 |
243 | for j in range(len_decode + 1):
244 | costs[curr][j] = j
245 | for i in range(1, len_text + 1):
246 | curr, prev = prev, curr
247 | costs[curr][0] = i
248 | ch = text[i - 1]
249 | if not cased:
250 | ch = ch.lower()
251 | for j in range(1, len_decode + 1):
252 | costs[curr][j] = costs[prev][j - 1]
253 | paths[i][j] = (i - 1, j - 1)
254 | if ch != decoded[j - 1]:
255 | costs[curr][j] = costs[prev][j - 1]
256 | paths[i][j] = (i - 1, j - 1)
257 | if costs[prev][j] < costs[curr][j]:
258 | costs[curr][j] = costs[prev][j]
259 | paths[i][j] = (i - 1, j)
260 | if costs[curr][j - 1] < costs[curr][j]:
261 | costs[curr][j] = costs[curr][j - 1]
262 | paths[i][j] = (i, j - 1)
263 | costs[curr][j] += 1
264 |
265 | matches = [0] * (len_decode + 1)
266 | position = (len_text, len_decode)
267 | while position != (-1, -1):
268 | i, j = position
269 | matches[j] = i
270 | position = paths[i][j]
271 |
272 | intervals = [[matches[offset[0]], matches[offset[1]]] for offset in token_offsets]
273 | for i, interval in enumerate(intervals):
274 | token_a, token_b = text[interval[0]:interval[1]], tokens[i]
275 | if len(token_b) > 2 and token_b.startswith('##'):
276 | token_b = token_b[2:]
277 | if not cased:
278 | token_a, token_b = token_a.lower(), token_b.lower()
279 | if token_a == token_b:
280 | continue
281 | if i == 0:
282 | border = 0
283 | else:
284 | border = intervals[i - 1][1]
285 | for j in range(interval[0] - 1, border - 1, -1):
286 | if Tokenizer._is_space(text[j]):
287 | break
288 | interval[0] -= 1
289 | if i + 1 == len(intervals):
290 | border = len_text
291 | else:
292 | border = intervals[i + 1][0]
293 | for j in range(interval[1], border):
294 | if Tokenizer._is_space(text[j]):
295 | break
296 | interval[1] += 1
297 | intervals = [(interval[0] + heading, interval[1] + heading) for interval in intervals]
298 | return intervals
299 |
300 |
--------------------------------------------------------------------------------
/xbert/xbert_model.py:
--------------------------------------------------------------------------------
1 | # !/usr/bin/python
2 | # -*- coding: utf-8 -*-
3 | #
4 | # Copyright @group xuyongfu. All Rights Reserved.
5 | #
6 | # @author: yuyangmu
7 | # @date: 2020/05/24
8 |
9 | # the function of this file: implement the main structure of transformer and bert based on tf2
10 |
11 |
12 | from .backend import keras, K
13 |
14 | from .layers.embedding_layer import Embedding
15 | from .layers.bias_add import BiasAdd
16 | from .layers.position_embedding import PositionEmbedding
17 | from .layers.multi_head_attention import MultiHeadAttention
18 | from .layers.layer_normalization import BasicLayerNormalization, ConditionalLayerNormalization
19 | from .layers.feed_forward import FeedForward
20 | from tensorflow.keras.layers import *
21 | from tensorflow.keras.models import Model
22 | import numpy as np
23 | import tensorflow as tf
24 | import json
25 | from .utils import is_sting
26 |
27 |
28 | __all__ = [
29 | 'get_custom_objects', 'set_custom_objects', 'build_xbert_model',
30 | ]
31 |
32 |
33 | def get_custom_objects() -> dict:
34 | return {
35 | 'Embedding': Embedding,
36 | 'BiasAdd': BiasAdd,
37 | 'MultiHeadAttention': MultiHeadAttention,
38 | 'BasicLayerNormalization': BasicLayerNormalization,
39 | 'ConditionalLayerNormalization': ConditionalLayerNormalization,
40 | 'PositionEmbedding': PositionEmbedding,
41 | 'FeedForward': FeedForward,
42 | }
43 |
44 |
45 | def set_custom_objects() -> None:
46 | for key, val in get_custom_objects().items():
47 | keras.utils.get_custom_objects()[key] = val
48 |
49 |
50 | class TransformerBlock(object):
51 | """定义xbert模型基类TransformerBlock
52 | """
53 | def __init__(
54 | self,
55 | vocab_size, # 词表大小
56 | hidden_size, # 编码维度
57 | num_hidden_layers, # Transformer总层数
58 | num_attention_heads, # Attention的头数
59 | intermediate_size, # FeedForward的隐层维度
60 | hidden_act, # FeedForward隐层的激活函数
61 | dropout_rate=None, # Dropout比例
62 | embedding_size=None, # 是否指定embedding_size
63 | attention_key_size=None, # Attention中Q,K的head_size
64 | sequence_length=None, # 是否固定序列长度
65 | keep_tokens=None, # 要保留的词ID列表
66 | layers=None, # 外部传入的Keras层
67 | name=None, # 模型名称
68 | **kwargs
69 | ):
70 | if keep_tokens is None:
71 | self.vocab_size = vocab_size
72 | else:
73 | self.vocab_size = len(keep_tokens)
74 | self.hidden_size = hidden_size
75 | self.num_hidden_layers = num_hidden_layers
76 | self.num_attention_heads = num_attention_heads
77 | self.attention_head_size = hidden_size // num_attention_heads
78 | self.attention_key_size = attention_key_size or self.attention_head_size
79 | self.intermediate_size = intermediate_size
80 | self.dropout_rate = dropout_rate or 0
81 | self.hidden_act = hidden_act
82 | self.embedding_size = embedding_size or hidden_size
83 | self.sequence_length = sequence_length
84 | self.keep_tokens = keep_tokens
85 | self.attention_mask = None
86 | self.position_bias = None
87 | self.layers = {} if layers is None else layers
88 | self.name = name
89 | self.built = False
90 |
91 | def build(
92 | self,
93 | layer_norm_cond=None,
94 | layer_norm_cond_hidden_size=None,
95 | layer_norm_cond_hidden_act=None,
96 | additional_input_layers=None,
97 | **kwargs
98 | ):
99 | """用于完整构建Transformer模型
100 | layer_norm_*系列参数为实现Conditional Layer Normalization时使用，
101 | 用来实现以“固定长度向量”为条件的条件Bert。
102 | """
103 | if self.built:
104 | return None
105 | # Input
106 | inputs = self.get_inputs()
107 | self.set_inputs(inputs, additional_input_layers)
108 | # Other
109 | self.layer_norm_conds = [
110 | layer_norm_cond,
111 | layer_norm_cond_hidden_size,
112 | layer_norm_cond_hidden_act or 'linear',
113 | ]
114 | # Call
115 | outputs = self.call(inputs)
116 | self.set_outputs(outputs)
117 | # Model
118 | self.model = Model(self.inputs, self.outputs, name=self.name)
119 | self.built = True
120 |
121 | def call(self, inputs):
122 | """模型的主要执行流程
123 | """
124 | # Embedding
125 | outputs = self.apply_embeddings(inputs)
126 | # Main
127 | for i in range(self.num_hidden_layers):
128 | outputs = self.apply_main_layers(outputs, i)
129 | # Final
130 | outputs = self.apply_final_layers(outputs)
131 | return outputs
132 |
133 | def apply(self, inputs, layer=None, arguments=None, **kwargs):
134 | """
135 | 通过apply调用层会自动重用同名层
136 | :param inputs: 上一层的输出；
137 | :param layer: 要调用的层类名；
138 | :param arguments: 传递给layer.call的参数；
139 | :param kwargs: 传递给层初始化的参数。
140 | """
141 | if layer is Dropout and self.dropout_rate == 0:
142 | return inputs
143 |
144 | arguments = arguments or {}
145 | name = kwargs.get('name')
146 | if name not in self.layers:
147 | layer = layer(**kwargs)
148 | name = layer.name
149 | self.layers[name] = layer
150 |
151 | return self.layers[name](inputs, **arguments)
152 |
153 | def get_inputs(self):
154 | raise NotImplementedError
155 |
156 | def apply_embeddings(self, inputs):
157 | raise NotImplementedError
158 |
159 | def apply_main_layers(self, inputs, index):
160 | raise NotImplementedError
161 |
162 | def apply_final_layers(self, inputs):
163 | raise NotImplementedError
164 |
165 | def compute_attention_mask(self, inputs=None):
166 | """定义每一层的Attention Mask
167 | """
168 | return self.attention_mask
169 |
170 | def compute_position_bias(self, inputs=None):
171 | """定义每一层的Position Bias（一般用在定义相对位置编码层）
172 | """
173 | return self.position_bias
174 |
175 | def set_inputs(self, inputs, additional_input_layers=None):
176 | """input和inputs属性设置
177 | """
178 | if inputs is None:
179 | inputs = []
180 | elif not isinstance(inputs, list):
181 | inputs = [inputs]
182 |
183 | inputs = inputs[:]
184 | if additional_input_layers is not None:
185 | if not isinstance(additional_input_layers, list):
186 | additional_input_layers = [additional_input_layers]
187 | inputs.extend(additional_input_layers)
188 |
189 | self.inputs = inputs
190 | if len(inputs) > 1:
191 | self.input = inputs
192 | else:
193 | self.input = inputs[0]
194 |
195 | def set_outputs(self, outputs):
196 | """output和oututs属性设置
197 | """
198 | if not isinstance(outputs, list):
199 | outputs = [outputs]
200 |
201 | outputs = outputs[:]
202 | self.outputs = outputs
203 | if len(outputs) > 1:
204 | self.output = outputs
205 | else:
206 | self.output = outputs[0]
207 |
208 | @property
209 | def initializer(self):
210 | """默认使用截断正态分布初始化
211 | """
212 | return keras.initializers.TruncatedNormal(stddev=0.02)
213 |
214 | def simplify(self, inputs):
215 | """将list中的None过滤掉
216 | """
217 | inputs = [i for i in inputs if i is not None]
218 | if len(inputs) == 1:
219 | inputs = inputs[0]
220 |
221 | return inputs
222 |
223 | def load_variable(self, checkpoint, name):
224 | """加载pre_model的checkpoint中单个变量
225 | """
226 | return tf.train.load_variable(checkpoint, name)
227 |
228 | def create_variable(self, name, value):
229 | """用tensorflow中创建可用变量
230 | """
231 | return tf.Variable(value, name=name)
232 |
233 | def variable_mapping(self):
234 | """构建tf.keras层与checkpoint的变量名之间的映射表
235 | """
236 | return {}
237 |
238 | def load_weights_from_checkpoint(self, checkpoint, mapping=None):
239 | """根据mapping从checkpoint加载权重
240 | """
241 | mapping = mapping or self.variable_mapping()
242 | mapping = {k: v for k, v in mapping.items() if k in self.layers}
243 |
244 | weight_value_pairs = []
245 | for layer, variables in mapping.items():
246 | layer = self.layers[layer]
247 | weights = layer.trainable_weights
248 | values = [self.load_variable(checkpoint, v) for v in variables]
249 |
250 | if isinstance(layer, MultiHeadAttention):
251 | """
252 | 如果key_size不等于head_size，则可以通过
253 | 正交矩阵将相应的权重投影到合适的shape。
254 | """
255 | count = 2
256 | if layer.use_bias:
257 | count += 2
258 | heads = self.num_attention_heads
259 | head_size = self.attention_head_size
260 | key_size = self.attention_key_size
261 | W = np.linalg.qr(np.random.randn(key_size, head_size))[0].T
262 | if layer.attention_scale:
263 | W = W * key_size**0.25 / head_size**0.25
264 | for i in range(count):
265 | w, v = weights[i], values[i]
266 | w_shape, v_shape = K.int_shape(w), v.shape
267 | if w_shape[-1] != v_shape[-1]:
268 | pre_shape = w_shape[:-1]
269 | v = v.reshape(pre_shape + (heads, head_size))
270 | v = np.dot(v, W)
271 | v = v.reshape(pre_shape + (heads * key_size,))
272 | values[i] = v
273 |
274 | weight_value_pairs.extend(zip(weights, values))
275 |
276 | K.batch_set_value(weight_value_pairs)
277 |
278 |
279 | class BERT(TransformerBlock):
280 | """通过transformer block构建BERT模型
281 | """
282 | def __init__(
283 | self,
284 | max_position, # 序列最大长度
285 | with_pool=False, # 是否包含Pool部分
286 | with_nsp=False, # 是否包含NSP部分
287 | with_mlm=False, # 是否包含MLM部分
288 | custom_position_ids=False, # 是否自行传入位置id
289 | **kwargs # 其余参数
290 | ):
291 | super(BERT, self).__init__(**kwargs)
292 | self.max_position = max_position
293 | self.with_pool = with_pool
294 | self.with_nsp = with_nsp
295 | self.with_mlm = with_mlm
296 | self.custom_position_ids = custom_position_ids
297 |
298 | def get_inputs(self):
299 | """
300 | token_ids和segment_ids为主要输入，
301 | （也允许自行传入位置id，以实现一些特殊需求）
302 | """
303 | x_in = Input(shape=(self.sequence_length,), name='Input_Token')
304 | s_in = Input(shape=(self.sequence_length,), name='Input_Segment')
305 |
306 | if self.custom_position_ids:
307 | p_in = Input(shape=(self.sequence_length,), name='Input_Position')
308 | return [x_in, s_in, p_in]
309 | else:
310 | return [x_in, s_in]
311 |
312 | def apply_embeddings(self, inputs):
313 | """BERT的embedding是token、position、segment三者embedding之和
314 | """
315 | x, s = inputs[:2]
316 | z = self.layer_norm_conds[0]
317 | if self.custom_position_ids:
318 | p = inputs[2]
319 | else:
320 | p = None
321 |
322 | x = self.apply(
323 | inputs=x,
324 | layer=Embedding,
325 | input_dim=self.vocab_size,
326 | output_dim=self.embedding_size,
327 | embeddings_initializer=self.initializer,
328 | mask_zero=True,
329 | name='Embedding-Token'
330 | )
331 | s = self.apply(
332 | inputs=s,
333 | layer=Embedding,
334 | input_dim=2,
335 | output_dim=self.embedding_size,
336 | embeddings_initializer=self.initializer,
337 | name='Embedding-Segment'
338 | )
339 | x = self.apply(inputs=[x, s], layer=Add, name='Embedding-Token-Segment')
340 | x = self.apply(
341 | inputs=self.simplify([x, p]),
342 | layer=PositionEmbedding,
343 | input_dim=self.max_position,
344 | output_dim=self.embedding_size,
345 | merge_mode='add',
346 | embeddings_initializer=self.initializer,
347 | custom_position_ids=self.custom_position_ids,
348 | name='Embedding-Position'
349 | )
350 | x = self.apply(
351 | inputs=self.simplify([x, z]),
352 | layer=ConditionalLayerNormalization,
353 | conditional=(z is not None),
354 | hidden_units=self.layer_norm_conds[1],
355 | hidden_activation=self.layer_norm_conds[2],
356 | hidden_initializer=self.initializer,
357 | name='Embedding-Norm'
358 | )
359 | x = self.apply(
360 | inputs=x,
361 | layer=Dropout,
362 | rate=self.dropout_rate,
363 | name='Embedding-Dropout'
364 | )
365 | if self.embedding_size != self.hidden_size:
366 | x = self.apply(
367 | inputs=x,
368 | layer=Dense,
369 | units=self.hidden_size,
370 | kernel_initializer=self.initializer,
371 | name='Embedding-Mapping'
372 | )
373 |
374 | return x
375 |
376 | def apply_main_layers(self, inputs, index):
377 | """
378 | BERT的主体是基于Self-Attention的模块
379 | 顺序：Att --> Add --> LN --> FFN --> Add --> LN
380 | """
381 | x = inputs
382 | z = self.layer_norm_conds[0]
383 |
384 | attention_name = 'Transformer-%d-MultiHeadSelfAttention' % index
385 | feed_forward_name = 'Transformer-%d-FeedForward' % index
386 | attention_mask = self.compute_attention_mask(index)
387 |
388 | # Self Attention
389 | xi, x, arguments = x, [x, x, x], {'a_mask': None}
390 | if attention_mask is not None:
391 | arguments['a_mask'] = True
392 | x.append(attention_mask)
393 |
394 | x = self.apply(
395 | inputs=x,
396 | layer=MultiHeadAttention,
397 | arguments=arguments,
398 | num_heads=self.num_attention_heads,
399 | head_size=self.attention_head_size,
400 | key_size=self.attention_key_size,
401 | kernel_initializer=self.initializer,
402 | name=attention_name
403 | )
404 | x = self.apply(
405 | inputs=x,
406 | layer=Dropout,
407 | rate=self.dropout_rate,
408 | name='%s-Dropout' % attention_name
409 | )
410 | x = self.apply(
411 | inputs=[xi, x], layer=Add, name='%s-Add' % attention_name
412 | )
413 | x = self.apply(
414 | inputs=self.simplify([x, z]),
415 | layer=ConditionalLayerNormalization,
416 | conditional=(z is not None),
417 | hidden_units=self.layer_norm_conds[1],
418 | hidden_activation=self.layer_norm_conds[2],
419 | hidden_initializer=self.initializer,
420 | name='%s-Norm' % attention_name
421 | )
422 |
423 | # Feed Forward
424 | xi = x
425 | x = self.apply(
426 | inputs=x,
427 | layer=FeedForward,
428 | units=self.intermediate_size,
429 | activation=self.hidden_act,
430 | kernel_initializer=self.initializer,
431 | name=feed_forward_name
432 | )
433 | x = self.apply(
434 | inputs=x,
435 | layer=Dropout,
436 | rate=self.dropout_rate,
437 | name='%s-Dropout' % feed_forward_name
438 | )
439 | x = self.apply(
440 | inputs=[xi, x], layer=Add, name='%s-Add' % feed_forward_name
441 | )
442 | x = self.apply(
443 | inputs=self.simplify([x, z]),
444 | layer=ConditionalLayerNormalization,
445 | conditional=(z is not None),
446 | hidden_units=self.layer_norm_conds[1],
447 | hidden_activation=self.layer_norm_conds[2],
448 | hidden_initializer=self.initializer,
449 | name='%s-Norm' % feed_forward_name
450 | )
451 |
452 | return x
453 |
454 | def apply_final_layers(self, inputs):
455 | """由剩余参数决定输出的形式
456 | """
457 | x = inputs
458 | z = self.layer_norm_conds[0]
459 | outputs = [x]
460 |
461 | if self.with_pool or self.with_nsp:
462 | # Pooler部分——提取CLS向量
463 | x = outputs[0]
464 | x = self.apply(
465 | inputs=x,
466 | layer=Lambda,
467 | function=lambda x: x[:, 0],
468 | name='Pooler'
469 | )
470 | pool_activation = 'tanh' if self.with_pool is True else self.with_pool
471 | x = self.apply(
472 | inputs=x,
473 | layer=Dense,
474 | units=self.hidden_size,
475 | activation=pool_activation,
476 | kernel_initializer=self.initializer,
477 | name='Pooler-Dense'
478 | )
479 | if self.with_nsp:
480 | # Next Sentence Prediction部分
481 | x = self.apply(
482 | inputs=x,
483 | layer=Dense,
484 | units=2,
485 | activation='softmax',
486 | kernel_initializer=self.initializer,
487 | name='NSP-Proba'
488 | )
489 | outputs.append(x)
490 |
491 | if self.with_mlm:
492 | # Masked Language Model部分
493 | x = outputs[0]
494 | x = self.apply(
495 | inputs=x,
496 | layer=Dense,
497 | units=self.embedding_size,
498 | activation=self.hidden_act,
499 | kernel_initializer=self.initializer,
500 | name='MLM-Dense'
501 | )
502 | x = self.apply(
503 | inputs=self.simplify([x, z]),
504 | layer=ConditionalLayerNormalization,
505 | conditional=(z is not None),
506 | hidden_units=self.layer_norm_conds[1],
507 | hidden_activation=self.layer_norm_conds[2],
508 | hidden_initializer=self.initializer,
509 | name='MLM-Norm'
510 | )
511 | x = self.apply(
512 | inputs=x,
513 | layer=Embedding,
514 | arguments={'mode': 'dense'},
515 | name='Embedding-Token'
516 | )
517 | x = self.apply(inputs=x, layer=BiasAdd, name='MLM-Bias')
518 | mlm_activation = 'softmax' if self.with_mlm is True else self.with_mlm
519 | x = self.apply(
520 | inputs=x,
521 | layer=Activation,
522 | activation=mlm_activation,
523 | name='MLM-Activation'
524 | )
525 | outputs.append(x)
526 |
527 | if len(outputs) == 1:
528 | outputs = outputs[0]
529 | elif len(outputs) == 2:
530 | outputs = outputs[1]
531 | else:
532 | outputs = outputs[1:]
533 |
534 | return outputs
535 |
536 | def load_variable(self, checkpoint, name):
537 | """加载pre_model的checkpoint中单个变量
538 | """
539 | variable = super(BERT, self).load_variable(checkpoint, name)
540 | if name in [
541 | 'bert/embeddings/word_embeddings',
542 | 'cls/predictions/output_bias',
543 | ]:
544 | if self.keep_tokens is None:
545 | return variable
546 | else:
547 | return variable[self.keep_tokens]
548 | elif name == 'cls/seq_relationship/output_weights':
549 | return variable.T
550 | else:
551 | return variable
552 |
553 | def create_variable(self, name, value):
554 | """用tensorflow中创建可用变量——根据输出格式定义
555 | """
556 | if name == 'cls/seq_relationship/output_weights':
557 | value = value.T
558 | return super(BERT, self).create_variable(name, value)
559 |
560 | def variable_mapping(self):
561 | """对官方bert的checkpoint中权重按格式进行映射
562 | """
563 | mapping = {
564 | 'Embedding-Token': ['bert/embeddings/word_embeddings'],
565 | 'Embedding-Segment': ['bert/embeddings/token_type_embeddings'],
566 | 'Embedding-Position': ['bert/embeddings/position_embeddings'],
567 | 'Embedding-Norm': [
568 | 'bert/embeddings/LayerNorm/beta',
569 | 'bert/embeddings/LayerNorm/gamma',
570 | ],
571 | 'Embedding-Mapping': [
572 | 'bert/encoder/embedding_hidden_mapping_in/kernel',
573 | 'bert/encoder/embedding_hidden_mapping_in/bias',
574 | ],
575 | 'Pooler-Dense': [
576 | 'bert/pooler/dense/kernel',
577 | 'bert/pooler/dense/bias',
578 | ],
579 | 'NSP-Proba': [
580 | 'cls/seq_relationship/output_weights',
581 | 'cls/seq_relationship/output_bias',
582 | ],
583 | 'MLM-Dense': [
584 | 'cls/predictions/transform/dense/kernel',
585 | 'cls/predictions/transform/dense/bias',
586 | ],
587 | 'MLM-Norm': [
588 | 'cls/predictions/transform/LayerNorm/beta',
589 | 'cls/predictions/transform/LayerNorm/gamma',
590 | ],
591 | 'MLM-Bias': ['cls/predictions/output_bias'],
592 | }
593 |
594 | for i in range(self.num_hidden_layers):
595 | prefix = 'bert/encoder/layer_%d/' % i
596 | mapping.update({
597 | 'Transformer-%d-MultiHeadSelfAttention' % i: [
598 | prefix + 'attention/self/query/kernel',
599 | prefix + 'attention/self/query/bias',
600 | prefix + 'attention/self/key/kernel',
601 | prefix + 'attention/self/key/bias',
602 | prefix + 'attention/self/value/kernel',
603 | prefix + 'attention/self/value/bias',
604 | prefix + 'attention/output/dense/kernel',
605 | prefix + 'attention/output/dense/bias',
606 | ],
607 | 'Transformer-%d-MultiHeadSelfAttention-Norm' % i: [
608 | prefix + 'attention/output/LayerNorm/beta',
609 | prefix + 'attention/output/LayerNorm/gamma',
610 | ],
611 | 'Transformer-%d-FeedForward' % i: [
612 | prefix + 'intermediate/dense/kernel',
613 | prefix + 'intermediate/dense/bias',
614 | prefix + 'output/dense/kernel',
615 | prefix + 'output/dense/bias',
616 | ],
617 | 'Transformer-%d-FeedForward-Norm' % i: [
618 | prefix + 'output/LayerNorm/beta',
619 | prefix + 'output/LayerNorm/gamma',
620 | ],
621 | })
622 |
623 | return mapping
624 |
625 |
626 | # 调用此函数直接构造xbert模型
627 | def build_xbert_model(
628 | config_path=None,
629 | checkpoint_path=None,
630 | model='bert',
631 | application='encoder',
632 | return_keras_model=True,
633 | **kwargs
634 | ):
635 | """通过load bert's pre_trained checkpoint weights 和配置文件来直接构造bert模型
636 | :param config_path: # bert参数文件路径
637 | :param checkpoint_path: # bert预训练模型路径
638 | :param model: # 构建模型的类型名字，默认加载google官方bert
639 | :param application: # 默认使用encoder功能
640 | :param return_keras_model: # 默认返回tf.keras模型
641 | :param kwargs:
642 | """
643 | configs = {}
644 | if config_path is not None:
645 | configs.update(json.load(open(config_path)))
646 | configs.update(kwargs)
647 | if 'max_position' not in configs:
648 | configs['max_position'] = configs.get('max_position_embeddings')
649 | if 'dropout_rate' not in configs:
650 | configs['dropout_rate'] = configs.get('hidden_dropout_prob')
651 |
652 | application = application.lower()
653 |
654 | models = {
655 | 'bert': BERT,
656 | }
657 |
658 | if is_sting(model):
659 | model = model.lower()
660 | MODEL = models[model]
661 | else:
662 | MODEL = model
663 |
664 | transformer_block = MODEL(**configs)
665 | transformer_block.build(**configs)
666 |
667 | if checkpoint_path is not None:
668 | transformer_block.load_weights_from_checkpoint(checkpoint_path)
669 |
670 | if return_keras_model:
671 | return transformer_block.model
672 | else:
673 | return transformer_block
674 |
675 |
676 |
677 |
678 |
679 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | GNU GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 | Preamble
9 |
10 | The GNU General Public License is a free, copyleft license for
11 | software and other kinds of works.
12 |
13 | The licenses for most software and other practical works are designed
14 | to take away your freedom to share and change the works. By contrast,
15 | the GNU General Public License is intended to guarantee your freedom to
16 | share and change all versions of a program--to make sure it remains free
17 | software for all its users. We, the Free Software Foundation, use the
18 | GNU General Public License for most of our software; it applies also to
19 | any other work released this way by its authors. You can apply it to
20 | your programs, too.
21 |
22 | When we speak of free software, we are referring to freedom, not
23 | price. Our General Public Licenses are designed to make sure that you
24 | have the freedom to distribute copies of free software (and charge for
25 | them if you wish), that you receive source code or can get it if you
26 | want it, that you can change the software or use pieces of it in new
27 | free programs, and that you know you can do these things.
28 |
29 | To protect your rights, we need to prevent others from denying you
30 | these rights or asking you to surrender the rights. Therefore, you have
31 | certain responsibilities if you distribute copies of the software, or if
32 | you modify it: responsibilities to respect the freedom of others.
33 |
34 | For example, if you distribute copies of such a program, whether
35 | gratis or for a fee, you must pass on to the recipients the same
36 | freedoms that you received. You must make sure that they, too, receive
37 | or can get the source code. And you must show them these terms so they
38 | know their rights.
39 |
40 | Developers that use the GNU GPL protect your rights with two steps:
41 | (1) assert copyright on the software, and (2) offer you this License
42 | giving you legal permission to copy, distribute and/or modify it.
43 |
44 | For the developers' and authors' protection, the GPL clearly explains
45 | that there is no warranty for this free software. For both users' and
46 | authors' sake, the GPL requires that modified versions be marked as
47 | changed, so that their problems will not be attributed erroneously to
48 | authors of previous versions.
49 |
50 | Some devices are designed to deny users access to install or run
51 | modified versions of the software inside them, although the manufacturer
52 | can do so. This is fundamentally incompatible with the aim of
53 | protecting users' freedom to change the software. The systematic
54 | pattern of such abuse occurs in the area of products for individuals to
55 | use, which is precisely where it is most unacceptable. Therefore, we
56 | have designed this version of the GPL to prohibit the practice for those
57 | products. If such problems arise substantially in other domains, we
58 | stand ready to extend this provision to those domains in future versions
59 | of the GPL, as needed to protect the freedom of users.
60 |
61 | Finally, every program is threatened constantly by software patents.
62 | States should not allow patents to restrict development and use of
63 | software on general-purpose computers, but in those that do, we wish to
64 | avoid the special danger that patents applied to a free program could
65 | make it effectively proprietary. To prevent this, the GPL assures that
66 | patents cannot be used to render the program non-free.
67 |
68 | The precise terms and conditions for copying, distribution and
69 | modification follow.
70 |
71 | TERMS AND CONDITIONS
72 |
73 | 0. Definitions.
74 |
75 | "This License" refers to version 3 of the GNU General Public License.
76 |
77 | "Copyright" also means copyright-like laws that apply to other kinds of
78 | works, such as semiconductor masks.
79 |
80 | "The Program" refers to any copyrightable work licensed under this
81 | License. Each licensee is addressed as "you". "Licensees" and
82 | "recipients" may be individuals or organizations.
83 |
84 | To "modify" a work means to copy from or adapt all or part of the work
85 | in a fashion requiring copyright permission, other than the making of an
86 | exact copy. The resulting work is called a "modified version" of the
87 | earlier work or a work "based on" the earlier work.
88 |
89 | A "covered work" means either the unmodified Program or a work based
90 | on the Program.
91 |
92 | To "propagate" a work means to do anything with it that, without
93 | permission, would make you directly or secondarily liable for
94 | infringement under applicable copyright law, except executing it on a
95 | computer or modifying a private copy. Propagation includes copying,
96 | distribution (with or without modification), making available to the
97 | public, and in some countries other activities as well.
98 |
99 | To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies. Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 |
103 | An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License. If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 |
112 | 1. Source Code.
113 |
114 | The "source code" for a work means the preferred form of the work
115 | for making modifications to it. "Object code" means any non-source
116 | form of a work.
117 |
118 | A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 |
123 | The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form. A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 |
134 | The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities. However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work. For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 |
147 | The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 |
151 | The Corresponding Source for a work in source code form is that
152 | same work.
153 |
154 | 2. Basic Permissions.
155 |
156 | All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met. This License explicitly affirms your unlimited
159 | permission to run the unmodified Program. The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work. This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 |
164 | You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force. You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright. Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 |
175 | Conveying under any other circumstances is permitted solely under
176 | the conditions stated below. Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 |
179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 |
181 | No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 |
187 | When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 |
195 | 4. Conveying Verbatim Copies.
196 |
197 | You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 |
205 | You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 |
208 | 5. Conveying Modified Source Versions.
209 |
210 | You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 |
214 | a) The work must carry prominent notices stating that you modified
215 | it, and giving a relevant date.
216 |
217 | b) The work must carry prominent notices stating that it is
218 | released under this License and any conditions added under section
219 | 7. This requirement modifies the requirement in section 4 to
220 | "keep intact all notices".
221 |
222 | c) You must license the entire work, as a whole, under this
223 | License to anyone who comes into possession of a copy. This
224 | License will therefore apply, along with any applicable section 7
225 | additional terms, to the whole of the work, and all its parts,
226 | regardless of how they are packaged. This License gives no
227 | permission to license the work in any other way, but it does not
228 | invalidate such permission if you have separately received it.
229 |
230 | d) If the work has interactive user interfaces, each must display
231 | Appropriate Legal Notices; however, if the Program has interactive
232 | interfaces that do not display Appropriate Legal Notices, your
233 | work need not make them do so.
234 |
235 | A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit. Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 |
245 | 6. Conveying Non-Source Forms.
246 |
247 | You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 |
252 | a) Convey the object code in, or embodied in, a physical product
253 | (including a physical distribution medium), accompanied by the
254 | Corresponding Source fixed on a durable physical medium
255 | customarily used for software interchange.
256 |
257 | b) Convey the object code in, or embodied in, a physical product
258 | (including a physical distribution medium), accompanied by a
259 | written offer, valid for at least three years and valid for as
260 | long as you offer spare parts or customer support for that product
261 | model, to give anyone who possesses the object code either (1) a
262 | copy of the Corresponding Source for all the software in the
263 | product that is covered by this License, on a durable physical
264 | medium customarily used for software interchange, for a price no
265 | more than your reasonable cost of physically performing this
266 | conveying of source, or (2) access to copy the
267 | Corresponding Source from a network server at no charge.
268 |
269 | c) Convey individual copies of the object code with a copy of the
270 | written offer to provide the Corresponding Source. This
271 | alternative is allowed only occasionally and noncommercially, and
272 | only if you received the object code with such an offer, in accord
273 | with subsection 6b.
274 |
275 | d) Convey the object code by offering access from a designated
276 | place (gratis or for a charge), and offer equivalent access to the
277 | Corresponding Source in the same way through the same place at no
278 | further charge. You need not require recipients to copy the
279 | Corresponding Source along with the object code. If the place to
280 | copy the object code is a network server, the Corresponding Source
281 | may be on a different server (operated by you or a third party)
282 | that supports equivalent copying facilities, provided you maintain
283 | clear directions next to the object code saying where to find the
284 | Corresponding Source. Regardless of what server hosts the
285 | Corresponding Source, you remain obligated to ensure that it is
286 | available for as long as needed to satisfy these requirements.
287 |
288 | e) Convey the object code using peer-to-peer transmission, provided
289 | you inform other peers where the object code and Corresponding
290 | Source of the work are being offered to the general public at no
291 | charge under subsection 6d.
292 |
293 | A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 |
297 | A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling. In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage. For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product. A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 |
310 | "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source. The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 |
318 | If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information. But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 |
329 | The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed. Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 |
337 | Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 |
343 | 7. Additional Terms.
344 |
345 | "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law. If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 |
354 | When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it. (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.) You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 |
361 | Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 |
365 | a) Disclaiming warranty or limiting liability differently from the
366 | terms of sections 15 and 16 of this License; or
367 |
368 | b) Requiring preservation of specified reasonable legal notices or
369 | author attributions in that material or in the Appropriate Legal
370 | Notices displayed by works containing it; or
371 |
372 | c) Prohibiting misrepresentation of the origin of that material, or
373 | requiring that modified versions of such material be marked in
374 | reasonable ways as different from the original version; or
375 |
376 | d) Limiting the use for publicity purposes of names of licensors or
377 | authors of the material; or
378 |
379 | e) Declining to grant rights under trademark law for use of some
380 | trade names, trademarks, or service marks; or
381 |
382 | f) Requiring indemnification of licensors and authors of that
383 | material by anyone who conveys the material (or modified versions of
384 | it) with contractual assumptions of liability to the recipient, for
385 | any liability that these contractual assumptions directly impose on
386 | those licensors and authors.
387 |
388 | All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10. If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term. If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 |
398 | If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 |
403 | Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 |
407 | 8. Termination.
408 |
409 | You may not propagate or modify a covered work except as expressly
410 | provided under this License. Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 |
415 | However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 |
422 | Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 |
429 | Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License. If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 |
435 | 9. Acceptance Not Required for Having Copies.
436 |
437 | You are not required to accept this License in order to receive or
438 | run a copy of the Program. Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance. However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work. These actions infringe copyright if you do
443 | not accept this License. Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 |
446 | 10. Automatic Licensing of Downstream Recipients.
447 |
448 | Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License. You are not responsible
451 | for enforcing compliance by third parties with this License.
452 |
453 | An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations. If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 |
463 | You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License. For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 |
471 | 11. Patents.
472 |
473 | A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based. The
475 | work thus licensed is called the contributor's "contributor version".
476 |
477 | A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version. For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 |
487 | Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 |
492 | In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement). To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 |
499 | If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients. "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 |
513 | If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 |
521 | A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License. You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 |
536 | Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 |
540 | 12. No Surrender of Others' Freedom.
541 |
542 | If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License. If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all. For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 |
552 | 13. Use with the GNU Affero General Public License.
553 |
554 | Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work. The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 |
563 | 14. Revised Versions of this License.
564 |
565 | The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time. Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 |
570 | Each version is given a distinguishing version number. If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation. If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 |
579 | If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 |
584 | Later license versions may give you additional or different
585 | permissions. However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 |
589 | 15. Disclaimer of Warranty.
590 |
591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 |
600 | 16. Limitation of Liability.
601 |
602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 |
612 | 17. Interpretation of Sections 15 and 16.
613 |
614 | If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 |
621 | END OF TERMS AND CONDITIONS
622 |
623 | How to Apply These Terms to Your New Programs
624 |
625 | If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 |
629 | To do so, attach the following notices to the program. It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 |
634 |
635 | Copyright (C)
636 |
637 | This program is free software: you can redistribute it and/or modify
638 | it under the terms of the GNU General Public License as published by
639 | the Free Software Foundation, either version 3 of the License, or
640 | (at your option) any later version.
641 |
642 | This program is distributed in the hope that it will be useful,
643 | but WITHOUT ANY WARRANTY; without even the implied warranty of
644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645 | GNU General Public License for more details.
646 |
647 | You should have received a copy of the GNU General Public License
648 | along with this program. If not, see .
649 |
650 | Also add information on how to contact you by electronic and paper mail.
651 |
652 | If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 |
655 | Copyright (C)
656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 | This is free software, and you are welcome to redistribute it
658 | under certain conditions; type `show c' for details.
659 |
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License. Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 |
664 | You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | .
668 |
669 | The GNU General Public License does not permit incorporating your program
670 | into proprietary programs. If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library. If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License. But first, please read
674 | .
675 |
--------------------------------------------------------------------------------