pytorch | Dynamic neural networks in Python with strong GPU | Machine Learning library

a = [1, 2, 3]
b = a

def __call__(self, input1):
    self.my_function(input1)

__call__ = my_function

a = [1, 2, 3]
b = a

def __call__(self, input1):
    self.my_function(input1)

__call__ = my_function

a = [1, 2, 3]
b = a

def __call__(self, input1):
    self.my_function(input1)

__call__ = my_function

def is_picklable(obj):
  try:
    pickle.dumps(obj)

  except pickle.PicklingError:
    return False
  return True

def is_picklable(obj: Any) -> bool:
    try:
        pickle.dumps(obj)
        return True
    except (pickle.PicklingError, pickle.PickleError, AttributeError, ImportError):
        # https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled
        return False
    except RecursionError:
        warnings.warn(
            f"Could not determine if object of type {type(obj)!r} is picklable"
            "due to a RecursionError that was supressed. "
            "Setting a higher recursion limit MAY allow this object to be pickled"
        )
        return False
    except Exception as e:
        # https://docs.python.org/3/library/pickle.html#id9
        warnings.warn(
            f"An error occurred while attempting to pickle"
            f"object of type {type(obj)!r}. Assuming it's unpicklable. The exception was {e}"
        )
        return False

class Unpicklable:
    """
    A simple marker class so we can distinguish when a deserialized object
    is a string because it was originally unpicklable 
    (and not simply a string to begin with)
    """
    def __init__(self, obj_str: str):
        self.obj_str = obj_str

    def __str__(self):
        return self.obj_str

    def __repr__(self):
        return f'Unpicklable(obj_str={self.obj_str!r})'


class PicklableNamespace(Namespace):
    def __getstate__(self):
        """For serialization"""

        # always make a copy so you don't accidentally modify state
        state = self.__dict__.copy()

        # Any unpicklables will be converted to a ``Unpicklable`` object 
        # with its str format stored in the object
        for key, val in state.items():
            if not is_picklable(val):
                state[key] = Unpicklable(str(val))
        return state
    def __setstate__(self, state):
        self.__dict__.update(state)  # or leave unimplemented

# Normally file handles are not picklable
p = PicklableNamespace(f=open('test.txt'))

data = pickle.dumps(p)
del p

loaded_p = pickle.loads(data)
# PicklableNamespace(f=Unpicklable(obj_str="<_io.TextIOWrapper name='test.txt' mode='r' encoding='cp1252'>"))

def is_picklable(obj: Any) -> bool:
    try:
        pickle.dumps(obj)
        return True
    except (pickle.PicklingError, pickle.PickleError, AttributeError, ImportError):
        # https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled
        return False
    except RecursionError:
        warnings.warn(
            f"Could not determine if object of type {type(obj)!r} is picklable"
            "due to a RecursionError that was supressed. "
            "Setting a higher recursion limit MAY allow this object to be pickled"
        )
        return False
    except Exception as e:
        # https://docs.python.org/3/library/pickle.html#id9
        warnings.warn(
            f"An error occurred while attempting to pickle"
            f"object of type {type(obj)!r}. Assuming it's unpicklable. The exception was {e}"
        )
        return False

class Unpicklable:
    """
    A simple marker class so we can distinguish when a deserialized object
    is a string because it was originally unpicklable 
    (and not simply a string to begin with)
    """
    def __init__(self, obj_str: str):
        self.obj_str = obj_str

    def __str__(self):
        return self.obj_str

    def __repr__(self):
        return f'Unpicklable(obj_str={self.obj_str!r})'


class PicklableNamespace(Namespace):
    def __getstate__(self):
        """For serialization"""

        # always make a copy so you don't accidentally modify state
        state = self.__dict__.copy()

        # Any unpicklables will be converted to a ``Unpicklable`` object 
        # with its str format stored in the object
        for key, val in state.items():
            if not is_picklable(val):
                state[key] = Unpicklable(str(val))
        return state
    def __setstate__(self, state):
        self.__dict__.update(state)  # or leave unimplemented

# Normally file handles are not picklable
p = PicklableNamespace(f=open('test.txt'))

data = pickle.dumps(p)
del p

loaded_p = pickle.loads(data)
# PicklableNamespace(f=Unpicklable(obj_str="<_io.TextIOWrapper name='test.txt' mode='r' encoding='cp1252'>"))

def is_picklable(obj: Any) -> bool:
    try:
        pickle.dumps(obj)
        return True
    except (pickle.PicklingError, pickle.PickleError, AttributeError, ImportError):
        # https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled
        return False
    except RecursionError:
        warnings.warn(
            f"Could not determine if object of type {type(obj)!r} is picklable"
            "due to a RecursionError that was supressed. "
            "Setting a higher recursion limit MAY allow this object to be pickled"
        )
        return False
    except Exception as e:
        # https://docs.python.org/3/library/pickle.html#id9
        warnings.warn(
            f"An error occurred while attempting to pickle"
            f"object of type {type(obj)!r}. Assuming it's unpicklable. The exception was {e}"
        )
        return False

class Unpicklable:
    """
    A simple marker class so we can distinguish when a deserialized object
    is a string because it was originally unpicklable 
    (and not simply a string to begin with)
    """
    def __init__(self, obj_str: str):
        self.obj_str = obj_str

    def __str__(self):
        return self.obj_str

    def __repr__(self):
        return f'Unpicklable(obj_str={self.obj_str!r})'


class PicklableNamespace(Namespace):
    def __getstate__(self):
        """For serialization"""

        # always make a copy so you don't accidentally modify state
        state = self.__dict__.copy()

        # Any unpicklables will be converted to a ``Unpicklable`` object 
        # with its str format stored in the object
        for key, val in state.items():
            if not is_picklable(val):
                state[key] = Unpicklable(str(val))
        return state
    def __setstate__(self, state):
        self.__dict__.update(state)  # or leave unimplemented

# Normally file handles are not picklable
p = PicklableNamespace(f=open('test.txt'))

data = pickle.dumps(p)
del p

loaded_p = pickle.loads(data)
# PicklableNamespace(f=Unpicklable(obj_str="<_io.TextIOWrapper name='test.txt' mode='r' encoding='cp1252'>"))

def is_picklable(obj: Any) -> bool:
    """
    Checks if somehting is pickable.

    Ref:
        - https://stackoverflow.com/questions/70128335/what-is-the-proper-way-to-make-an-object-with-unpickable-fields-pickable
        - pycharm halting all the time issue: https://stackoverflow.com/questions/70761481/how-to-stop-pycharms-break-stop-halt-feature-on-handled-exceptions-i-e-only-b
    """
    import pickle
    try:
        pickle.dumps(obj)
    except:
        return False
    return True

import networkx as nx
import numpy as np
import torch
from torch_geometric.utils.convert import from_networkx


# Make the networkx graph
G = nx.Graph()

# Add some cars (just do 4 for now)
G.add_nodes_from([
      (1, {'y': 1, 'x': 0.5}),
      (2, {'y': 2, 'x': 0.2}),
      (3, {'y': 3, 'x': 0.3}),
      (4, {'y': 4, 'x': 0.1}),
      (5, {'y': 5, 'x': 0.2}),
])

# Add some edges
G.add_edges_from([
                  (1, 2), (1, 4), (1, 5),
                  (2, 3), (2, 4),
                  (3, 2), (3, 5),
                  (4, 1), (4, 2),
                  (5, 1), (5, 3)
])

# Convert the graph into PyTorch geometric
pyg_graph = from_networkx(G)

print(pyg_graph)
# Data(edge_index=[2, 12], x=[5], y=[5])
print(pyg_graph.x)
# tensor([0.5000, 0.2000, 0.3000, 0.1000, 0.2000])
print(pyg_graph.y)
# tensor([1, 2, 3, 4, 5])
print(pyg_graph.edge_index)
# tensor([[0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4],
#         [1, 3, 4, 0, 2, 3, 1, 4, 0, 1, 0, 2]])


# Split the data 
train_ratio = 0.2
num_nodes = pyg_graph.x.shape[0]
num_train = int(num_nodes * train_ratio)
idx = [i for i in range(num_nodes)]

np.random.shuffle(idx)
train_mask = torch.full_like(pyg_graph.y, False, dtype=bool)
train_mask[idx[:num_train]] = True
test_mask = torch.full_like(pyg_graph.y, False, dtype=bool)
test_mask[idx[num_train:]] = True

print(train_mask)
# tensor([ True, False, False, False, False])
print(test_mask)
# tensor([False,  True,  True,  True,  True])

self.flatten = nn.Flatten()
self.linear_relu_stack = nn.Sequential(
    nn.Linear(28*28, 512),
    nn.ReLU(),
    nn.Linear(512, 128),
    nn.ReLU(),
    nn.Linear(128, 10),
    nn.ReLU()
    )

out = (x - 1)s - 2p + d(k - 1) + op + 1

import collections
import logging
import os
import pathlib
import re
import string
import sys
import time
import numpy as np
#import matplotlib.pyplot as plt

import tensorflow_datasets as tfds
import tensorflow_text as text
import tensorflow as tf
from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab



examples, metadata = tfds.load('wmt14_translate/de-en', with_info=True,
                               as_supervised=True)
train_examples, val_examples = examples['train'], examples['validation']

for de_examples, en_examples in train_examples.batch(3).take(1):
  for pt in de_examples.numpy():
    print(pt.decode('utf-8'))

  print()

  for en in en_examples.numpy():
    print(en.decode('utf-8'))

train_en = train_examples.map(lambda de, en: en)
train_de = train_examples.map(lambda de, en: de)

bert_tokenizer_params=dict(lower_case=True)
reserved_tokens=["[PAD]", "[UNK]", "[START]", "[END]"]

bert_vocab_args = dict(
    # The target vocabulary size
    vocab_size = 8000,
    # Reserved tokens that must be included in the vocabulary
    reserved_tokens=reserved_tokens,
    # Arguments for `text.BertTokenizer`
    bert_tokenizer_params=bert_tokenizer_params,
    # Arguments for `wordpiece_vocab.wordpiece_tokenizer_learner_lib.learn`
    learn_params={},
)

de_vocab = bert_vocab.bert_vocab_from_dataset(
    train_de.batch(1000).prefetch(2),
    **bert_vocab_args
)

print(de_vocab[:10])
print(de_vocab[100:110])
print(de_vocab[1000:1010])
print(de_vocab[-10:])

def write_vocab_file(filepath, vocab):
  with open(filepath, 'w') as f:
    for token in vocab:
      print(token, file=f)

write_vocab_file('de_vocab.txt', de_vocab)

en_vocab = bert_vocab.bert_vocab_from_dataset(
    train_en.batch(1000).prefetch(2),
    **bert_vocab_args
)

print(en_vocab[:10])
print(en_vocab[100:110])
print(en_vocab[1000:1010])
print(en_vocab[-10:])

write_vocab_file('en_vocab.txt', en_vocab)

de_tokenizer = text.BertTokenizer('de_vocab.txt', **bert_tokenizer_params)
en_tokenizer = text.BertTokenizer('en_vocab.txt', **bert_tokenizer_params)

# Tokenize the examples -> (batch, word, word-piece)
token_batch = en_tokenizer.tokenize(en_examples)
# Merge the word and word-piece axes -> (batch, tokens)
token_batch = token_batch.merge_dims(-2,-1)

for ex in token_batch.to_list():
  print(ex)

# Lookup each token id in the vocabulary.
txt_tokens = tf.gather(en_vocab, token_batch)
# Join with spaces.
tf.strings.reduce_join(txt_tokens, separator=' ', axis=-1)

words = en_tokenizer.detokenize(token_batch)
tf.strings.reduce_join(words, separator=' ', axis=-1)

START = tf.argmax(tf.constant(reserved_tokens) == "[START]")
END = tf.argmax(tf.constant(reserved_tokens) == "[END]")

def add_start_end(ragged):
  count = ragged.bounding_shape()[0]
  starts = tf.fill([count,1], START)
  ends = tf.fill([count,1], END)
  return tf.concat([starts, ragged, ends], axis=1)

words = en_tokenizer.detokenize(add_start_end(token_batch))
tf.strings.reduce_join(words, separator=' ', axis=-1)

def cleanup_text(reserved_tokens, token_txt):
  # Drop the reserved tokens, except for "[UNK]".
  bad_tokens = [re.escape(tok) for tok in reserved_tokens if tok != "[UNK]"]
  bad_token_re = "|".join(bad_tokens)

  bad_cells = tf.strings.regex_full_match(token_txt, bad_token_re)
  result = tf.ragged.boolean_mask(token_txt, ~bad_cells)

  # Join them into strings.
  result = tf.strings.reduce_join(result, separator=' ', axis=-1)

  return result

token_batch = en_tokenizer.tokenize(en_examples).merge_dims(-2,-1)
words = en_tokenizer.detokenize(token_batch)

cleanup_text(reserved_tokens, words).numpy()

class CustomTokenizer(tf.Module):
  def __init__(self, reserved_tokens, vocab_path):
    self.tokenizer = text.BertTokenizer(vocab_path, lower_case=True)
    self._reserved_tokens = reserved_tokens
    self._vocab_path = tf.saved_model.Asset(vocab_path)

    vocab = pathlib.Path(vocab_path).read_text().splitlines()
    self.vocab = tf.Variable(vocab)

    ## Create the signatures for export:

    # Include a tokenize signature for a batch of strings.
    self.tokenize.get_concrete_function(
        tf.TensorSpec(shape=[None], dtype=tf.string))

    # Include `detokenize` and `lookup` signatures for:
    #   * `Tensors` with shapes [tokens] and [batch, tokens]
    #   * `RaggedTensors` with shape [batch, tokens]
    self.detokenize.get_concrete_function(
        tf.TensorSpec(shape=[None, None], dtype=tf.int64))
    self.detokenize.get_concrete_function(
          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))

    self.lookup.get_concrete_function(
        tf.TensorSpec(shape=[None, None], dtype=tf.int64))
    self.lookup.get_concrete_function(
          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))

    # These `get_*` methods take no arguments
    self.get_vocab_size.get_concrete_function()
    self.get_vocab_path.get_concrete_function()
    self.get_reserved_tokens.get_concrete_function()

  @tf.function
  def tokenize(self, strings):
    enc = self.tokenizer.tokenize(strings)
    # Merge the `word` and `word-piece` axes.
    enc = enc.merge_dims(-2,-1)
    enc = add_start_end(enc)
    return enc

  @tf.function
  def detokenize(self, tokenized):
    words = self.tokenizer.detokenize(tokenized)
    return cleanup_text(self._reserved_tokens, words)

  @tf.function
  def lookup(self, token_ids):
    return tf.gather(self.vocab, token_ids)

  @tf.function
  def get_vocab_size(self):
    return tf.shape(self.vocab)[0]

  @tf.function
  def get_vocab_path(self):
    return self._vocab_path

  @tf.function
  def get_reserved_tokens(self):
    return tf.constant(self._reserved_tokens)

tokenizers = tf.Module()
tokenizers.pt = CustomTokenizer(reserved_tokens, 'de_vocab.txt')
tokenizers.en = CustomTokenizer(reserved_tokens, 'en_vocab.txt')

model_name = 'ted_hrlr_translate_de_en_converter'
tf.saved_model.save(tokenizers, model_name)

import transformers 
import tensorflow as tf 
from tensorflow import keras

tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased")

tokenizer(["hello world", "this is me", "hello again"], return_tensors="tf", padding=True, truncation=True)

{'input_ids': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[ 101, 7592, 2088,  102,    0],
       [ 101, 2023, 2003, 2033,  102],
       [ 101, 7592, 2153,  102,    0]], dtype=int32)>, 'token_type_ids': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[1, 1, 1, 1, 0],
       [1, 1, 1, 1, 1],
       [1, 1, 1, 1, 0]], dtype=int32)>}

import transformers 
import tensorflow as tf 
from tensorflow import keras

tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-uncased")

tokenizer(["hello world", "this is me", "hello again"], return_tensors="tf", padding=True, truncation=True)

{'input_ids': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[ 101, 7592, 2088,  102,    0],
       [ 101, 2023, 2003, 2033,  102],
       [ 101, 7592, 2153,  102,    0]], dtype=int32)>, 'token_type_ids': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(3, 5), dtype=int32, numpy=
array([[1, 1, 1, 1, 0],
       [1, 1, 1, 1, 1],
       [1, 1, 1, 1, 0]], dtype=int32)>}

# in onnx NegativeLogLikelihoodLoss specification, ignore_index is optional without default value.
# therefore we need to set ignore_index attribute even if it is not specified (e.g. ignore_index=-100).
ignore_index = sym_help._maybe_get_const(ignore_index, "i")

df = pd.DataFrame(
    {
        "ID": [1, 2, 3],
        "Source": ["Serbia", "Poland", "Germany"],
        "Sequence": ["ABCDE", "EBCDA", "AAD"],
    }
)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, df: pd.DataFrame):
        self.df = df
        # Change alphabet to anything you need
        alphabet = ["A", "B", "C", "D", "E", "F"]
        self.mapping = dict((c, i) for i, c in enumerate(alphabet))

    def __getitem__(self, index):
        sample = df.iloc[index]
        sequence = sample["Sequence"]
        target = torch.nn.functional.one_hot(
            torch.tensor([self.mapping[letter] for letter in sequence]),
            num_classes=len(self.mapping),
        )
        return sample.drop("Sequence"), target

    def __len__(self):
        return len(self.df)

ds = Dataset(df)
ds[0]

tensor([ [1., 0., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0., 0.],
         [0., 0., 1., 0., 0., 0.],
         [0., 0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1., 0.]]))

df = pd.DataFrame(
    {
        "ID": [1, 2, 3],
        "Source": ["Serbia", "Poland", "Germany"],
        "Sequence": ["ABCDE", "EBCDA", "AAD"],
    }
)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, df: pd.DataFrame):
        self.df = df
        # Change alphabet to anything you need
        alphabet = ["A", "B", "C", "D", "E", "F"]
        self.mapping = dict((c, i) for i, c in enumerate(alphabet))

    def __getitem__(self, index):
        sample = df.iloc[index]
        sequence = sample["Sequence"]
        target = torch.nn.functional.one_hot(
            torch.tensor([self.mapping[letter] for letter in sequence]),
            num_classes=len(self.mapping),
        )
        return sample.drop("Sequence"), target

    def __len__(self):
        return len(self.df)

ds = Dataset(df)
ds[0]

tensor([ [1., 0., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0., 0.],
         [0., 0., 1., 0., 0., 0.],
         [0., 0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1., 0.]]))

df = pd.DataFrame(
    {
        "ID": [1, 2, 3],
        "Source": ["Serbia", "Poland", "Germany"],
        "Sequence": ["ABCDE", "EBCDA", "AAD"],
    }
)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, df: pd.DataFrame):
        self.df = df
        # Change alphabet to anything you need
        alphabet = ["A", "B", "C", "D", "E", "F"]
        self.mapping = dict((c, i) for i, c in enumerate(alphabet))

    def __getitem__(self, index):
        sample = df.iloc[index]
        sequence = sample["Sequence"]
        target = torch.nn.functional.one_hot(
            torch.tensor([self.mapping[letter] for letter in sequence]),
            num_classes=len(self.mapping),
        )
        return sample.drop("Sequence"), target

    def __len__(self):
        return len(self.df)

ds = Dataset(df)
ds[0]

tensor([ [1., 0., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0., 0.],
         [0., 0., 1., 0., 0., 0.],
         [0., 0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1., 0.]]))

df = pd.DataFrame(
    {
        "ID": [1, 2, 3],
        "Source": ["Serbia", "Poland", "Germany"],
        "Sequence": ["ABCDE", "EBCDA", "AAD"],
    }
)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, df: pd.DataFrame):
        self.df = df
        # Change alphabet to anything you need
        alphabet = ["A", "B", "C", "D", "E", "F"]
        self.mapping = dict((c, i) for i, c in enumerate(alphabet))

    def __getitem__(self, index):
        sample = df.iloc[index]
        sequence = sample["Sequence"]
        target = torch.nn.functional.one_hot(
            torch.tensor([self.mapping[letter] for letter in sequence]),
            num_classes=len(self.mapping),
        )
        return sample.drop("Sequence"), target

    def __len__(self):
        return len(self.df)

ds = Dataset(df)
ds[0]

tensor([ [1., 0., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0., 0.],
         [0., 0., 1., 0., 0., 0.],
         [0., 0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1., 0.]]))

data[i][index[i][j]] = 42

>>> data.scatter_(index=indices, dim=1, value=42)
>>> data
tensor([[ 0, 42,  2],
        [ 3,  4, 42],
        [ 6, 42,  8]])

data[i][index[i][j]] = 42

>>> data.scatter_(index=indices, dim=1, value=42)
>>> data
tensor([[ 0, 42,  2],
        [ 3,  4, 42],
        [ 6, 42,  8]])

import torch

# creating a dummy model
class Classifier(torch.nn.Module):
  def __init__(self):
    super(Classifier, self).__init__()
    self.first = torch.nn.Linear(10, 10)
    self.second = torch.nn.Linear(10, 20)
    self.last = torch.nn.Linear(20, 1)
  
  def forward(self, x):
    pass

# Creating its object
model = Classifier()

# Extracting the layer to save
to_save = model.last

# Saving the state dict of that layer
torch.save(to_save.state_dict(), './classifier.bin')

# Recreating the object of that model
model = Classifier()

# Updating the saved layer of model
model.last.load_state_dict(torch.load('./classifier.bin'))