Generator

1import numpy as np
2from timeit import timeit
3
4N = 10_000_000
5repetition = 10
6
7def sum0(N = N):
8    s = 0
9    i = 0
10    while i < N: # condition is checked in python
11        s += i
12        i += 1 # both additions are done in python
13    return s
14
15def sum1(N = N):
16    s = 0
17    for i in range(N): # increment in C
18        s += i # addition in python
19    return s
20
21def sum2(N = N):
22    return sum(range(N)) # everything in C
23
24def sum3(N = N):
25    return sum(list(range(N)))
26
27def sum4(N = N):
28    return np.sum(range(N)) # very slow np.array conversion
29
30def sum5(N = N):
31    # much faster np.array conversion
32    return np.sum(np.fromiter(range(N),dtype = int))
33
34def sum5v2_(N = N):
35    # much faster np.array conversion
36    return np.sum(np.fromiter(range(N),dtype = np.int_))
37
38def sum6(N = N):
39    # possibly slow conversion to Py_long from np.int
40    return sum(np.arange(N))
41
42def sum7(N = N):
43    # list returns a list of np.int-s
44    return sum(list(np.arange(N)))
45
46def sum7v2(N = N):
47    # tolist conversion to python int seems faster than the implicit conversion
48    # in sum(list()) (tolist returns a list of python int-s)
49    return sum(np.arange(N).tolist())
50
51def sum8(N = N):
52    return np.sum(np.arange(N)) # everything in numpy (fortran libblas?)
53
54def sum9(N = N):
55    return np.arange(N).sum() # remove dispatch overhead
56
57def array_basic(N = N):
58    return np.array(range(N))
59
60def array_dtype(N = N):
61    return np.array(range(N),dtype = np.int_)
62
63def array_iter(N = N):
64    # np.sum's source code mentions to use fromiter to convert from generators
65    return np.fromiter(range(N),dtype = np.int_)
66
67print(f"while loop:         {timeit(sum0, number = repetition)}")
68print(f"for loop:           {timeit(sum1, number = repetition)}")
69print(f"sum_range:          {timeit(sum2, number = repetition)}")
70print(f"sum_rangelist:      {timeit(sum3, number = repetition)}")
71print(f"npsum_range:        {timeit(sum4, number = repetition)}")
72print(f"npsum_iterrange:    {timeit(sum5, number = repetition)}")
73print(f"npsum_iterrangev2:  {timeit(sum5, number = repetition)}")
74print(f"sum_arange:         {timeit(sum6, number = repetition)}")
75print(f"sum_list_arange:    {timeit(sum7, number = repetition)}")
76print(f"sum_arange_tolist:  {timeit(sum7v2, number = repetition)}")
77print(f"npsum_arange:       {timeit(sum8, number = repetition)}")
78print(f"nparangenpsum:      {timeit(sum9, number = repetition)}")
79print(f"array_basic:        {timeit(array_basic, number = repetition)}")
80print(f"array_dtype:        {timeit(array_dtype, number = repetition)}")
81print(f"array_iter:         {timeit(array_iter,  number = repetition)}")
82
83print(f"npsumarangeREP:     {timeit(lambda : sum8(N/1000), number = 100000*repetition)}")
84print(f"npsumarangeREP:     {timeit(lambda : sum9(N/1000), number = 100000*repetition)}")
85
86# Example output:
87#
88# while loop:         11.493371912998555
89# for loop:           7.385945574002108
90# sum_range:          2.4605720699983067
91# sum_rangelist:      4.509678105998319
92# npsum_range:        11.85120212900074
93# npsum_iterrange:    4.464334709002287
94# npsum_iterrangev2:  4.498494338993623
95# sum_arange:         9.537815956995473
96# sum_list_arange:    13.290120724996086
97# sum_arange_tolist:  5.231948580003518
98# npsum_arange:       0.241889145996538
99# nparangenpsum:      0.21876695199898677
100# array_basic:        11.736577274998126
101# array_dtype:        8.71628468400013
102# array_iter:         4.303306431000237
103# npsumarangeREP:     21.240833958996518
104# npsumarangeREP:     16.690092379001726
105
106/* Fast addition by keeping temporary sums in C instead of new Python objects.
107   Assumes all inputs are the same type.  If the assumption fails, default
108   to the more general routine.
109*/
110

1import numpy as np
2from timeit import timeit
3
4N = 10_000_000
5repetition = 10
6
7def sum0(N = N):
8    s = 0
9    i = 0
10    while i < N: # condition is checked in python
11        s += i
12        i += 1 # both additions are done in python
13    return s
14
15def sum1(N = N):
16    s = 0
17    for i in range(N): # increment in C
18        s += i # addition in python
19    return s
20
21def sum2(N = N):
22    return sum(range(N)) # everything in C
23
24def sum3(N = N):
25    return sum(list(range(N)))
26
27def sum4(N = N):
28    return np.sum(range(N)) # very slow np.array conversion
29
30def sum5(N = N):
31    # much faster np.array conversion
32    return np.sum(np.fromiter(range(N),dtype = int))
33
34def sum5v2_(N = N):
35    # much faster np.array conversion
36    return np.sum(np.fromiter(range(N),dtype = np.int_))
37
38def sum6(N = N):
39    # possibly slow conversion to Py_long from np.int
40    return sum(np.arange(N))
41
42def sum7(N = N):
43    # list returns a list of np.int-s
44    return sum(list(np.arange(N)))
45
46def sum7v2(N = N):
47    # tolist conversion to python int seems faster than the implicit conversion
48    # in sum(list()) (tolist returns a list of python int-s)
49    return sum(np.arange(N).tolist())
50
51def sum8(N = N):
52    return np.sum(np.arange(N)) # everything in numpy (fortran libblas?)
53
54def sum9(N = N):
55    return np.arange(N).sum() # remove dispatch overhead
56
57def array_basic(N = N):
58    return np.array(range(N))
59
60def array_dtype(N = N):
61    return np.array(range(N),dtype = np.int_)
62
63def array_iter(N = N):
64    # np.sum's source code mentions to use fromiter to convert from generators
65    return np.fromiter(range(N),dtype = np.int_)
66
67print(f"while loop:         {timeit(sum0, number = repetition)}")
68print(f"for loop:           {timeit(sum1, number = repetition)}")
69print(f"sum_range:          {timeit(sum2, number = repetition)}")
70print(f"sum_rangelist:      {timeit(sum3, number = repetition)}")
71print(f"npsum_range:        {timeit(sum4, number = repetition)}")
72print(f"npsum_iterrange:    {timeit(sum5, number = repetition)}")
73print(f"npsum_iterrangev2:  {timeit(sum5, number = repetition)}")
74print(f"sum_arange:         {timeit(sum6, number = repetition)}")
75print(f"sum_list_arange:    {timeit(sum7, number = repetition)}")
76print(f"sum_arange_tolist:  {timeit(sum7v2, number = repetition)}")
77print(f"npsum_arange:       {timeit(sum8, number = repetition)}")
78print(f"nparangenpsum:      {timeit(sum9, number = repetition)}")
79print(f"array_basic:        {timeit(array_basic, number = repetition)}")
80print(f"array_dtype:        {timeit(array_dtype, number = repetition)}")
81print(f"array_iter:         {timeit(array_iter,  number = repetition)}")
82
83print(f"npsumarangeREP:     {timeit(lambda : sum8(N/1000), number = 100000*repetition)}")
84print(f"npsumarangeREP:     {timeit(lambda : sum9(N/1000), number = 100000*repetition)}")
85
86# Example output:
87#
88# while loop:         11.493371912998555
89# for loop:           7.385945574002108
90# sum_range:          2.4605720699983067
91# sum_rangelist:      4.509678105998319
92# npsum_range:        11.85120212900074
93# npsum_iterrange:    4.464334709002287
94# npsum_iterrangev2:  4.498494338993623
95# sum_arange:         9.537815956995473
96# sum_list_arange:    13.290120724996086
97# sum_arange_tolist:  5.231948580003518
98# npsum_arange:       0.241889145996538
99# nparangenpsum:      0.21876695199898677
100# array_basic:        11.736577274998126
101# array_dtype:        8.71628468400013
102# array_iter:         4.303306431000237
103# npsumarangeREP:     21.240833958996518
104# npsumarangeREP:     16.690092379001726
105
106/* Fast addition by keeping temporary sums in C instead of new Python objects.
107   Assumes all inputs are the same type.  If the assumption fails, default
108   to the more general routine.
109*/
110from functools import singledispatch
111
112def sum_range(range_, /, start=0):
113    """Overloaded `sum` for range, compute arithmetic sum"""
114    n = len(range_)
115    if not n:
116        return start
117    return int(start + (n * (range_[0] + range_[-1]) / 2))
118
119sum = singledispatch(sum)
120sum.register(range, sum_range)
121
122def test():
123    """
124    >>> sum(range(0, 100))
125    4950
126    >>> sum(range(0, 10, 2))
127    20
128    >>> sum(range(0, 9, 2))
129    20
130    >>> sum(range(0, -10, -1))
131    -45
132    >>> sum(range(-10, 10))
133    -10
134    >>> sum(range(-1, -100, -2))
135    -2500
136    >>> sum(range(0, 10, 100))
137    0
138    >>> sum(range(0, 0))
139    0
140    >>> sum(range(0, 100), 50)
141    5000
142    >>> sum(range(0, 0), 10)
143    10
144    """
145
146if __name__ == "__main__":
147    import doctest
148    doctest.testmod()
149

1#include <iostream>
2#include <ranges>
3#include <random>
4
5template<typename F>
6auto generate1(const F& func) {
7    return std::views::iota(0) | std::views::transform([&func](int) {return func(); });
8}
9
10std::random_device dev;
11std::mt19937 rng(dev());
12
13int main() {
14
15    auto d6 = []() {
16        static std::uniform_int_distribution<> dist(1, 6);
17        return dist(rng);
18    };
19
20    for (int v : generate1(d6) | std::views::take(10)) {
21        std::cout << v << ' ';
22    }
23    std::cout << '\n';
24}
25#include <iostream>
26#include <ranges>
27#include <random>
28
29template<typename F>
30class generate2 : public std::ranges::view_interface<generate2<F>>
31{
32    using value_type = decltype(std::declval<F>()());
33
34    class iterator {
35        const F* gen_func_;
36    public:
37        iterator(const F* f) : gen_func_(f)
38        {}
39
40        value_type operator*() const {
41            return (*gen_func_)();
42        }
43
44        bool operator!=(const iterator&) {
45            return true;
46        }
47
48        iterator& operator++() {
49            return *this;
50        }
51    };
52
53    F generator_func_;
54
55public:
56
57    generate2(const F& f) : generator_func_(f) {
58    }
59
60    iterator begin()  {
61        return iterator(&generator_func_);
62    }
63
64    iterator end()  {
65        return iterator(nullptr);
66    }
67};
68
69std::random_device dev;
70std::mt19937 rng(dev());
71
72int main() {
73
74    auto d6 = []() {
75        static std::uniform_int_distribution<> dist(1, 6);
76        return dist(rng);
77    };
78
79    // the following doesnt compile because of the pipe...
80    for (int v : generate2(d6) | std::views::take(10)) { 
81        std::cout << v << ' ';
82    }
83    std::cout << '\n';
84}
85class iterator {
86 public:
87  using value_type = decltype(std::declval<F>()());
88  using difference_type = std::ptrdiff_t;
89  iterator() = default;
90  void operator++(int);
91  bool operator==(const iterator&) const {
92    return false;
93  }
94  // ...
95};
96

1#include <iostream>
2#include <ranges>
3#include <random>
4
5template<typename F>
6auto generate1(const F& func) {
7    return std::views::iota(0) | std::views::transform([&func](int) {return func(); });
8}
9
10std::random_device dev;
11std::mt19937 rng(dev());
12
13int main() {
14
15    auto d6 = []() {
16        static std::uniform_int_distribution<> dist(1, 6);
17        return dist(rng);
18    };
19
20    for (int v : generate1(d6) | std::views::take(10)) {
21        std::cout << v << ' ';
22    }
23    std::cout << '\n';
24}
25#include <iostream>
26#include <ranges>
27#include <random>
28
29template<typename F>
30class generate2 : public std::ranges::view_interface<generate2<F>>
31{
32    using value_type = decltype(std::declval<F>()());
33
34    class iterator {
35        const F* gen_func_;
36    public:
37        iterator(const F* f) : gen_func_(f)
38        {}
39
40        value_type operator*() const {
41            return (*gen_func_)();
42        }
43
44        bool operator!=(const iterator&) {
45            return true;
46        }
47
48        iterator& operator++() {
49            return *this;
50        }
51    };
52
53    F generator_func_;
54
55public:
56
57    generate2(const F& f) : generator_func_(f) {
58    }
59
60    iterator begin()  {
61        return iterator(&generator_func_);
62    }
63
64    iterator end()  {
65        return iterator(nullptr);
66    }
67};
68
69std::random_device dev;
70std::mt19937 rng(dev());
71
72int main() {
73
74    auto d6 = []() {
75        static std::uniform_int_distribution<> dist(1, 6);
76        return dist(rng);
77    };
78
79    // the following doesnt compile because of the pipe...
80    for (int v : generate2(d6) | std::views::take(10)) { 
81        std::cout << v << ' ';
82    }
83    std::cout << '\n';
84}
85class iterator {
86 public:
87  using value_type = decltype(std::declval<F>()());
88  using difference_type = std::ptrdiff_t;
89  iterator() = default;
90  void operator++(int);
91  bool operator==(const iterator&) const {
92    return false;
93  }
94  // ...
95};
96template<typename F>
97class generate2 : public std::ranges::view_interface<generate2<F>> {
98 public:
99  auto begin() {
100    value_ = generator_func_();
101    return iterator{*this};
102  }
103
104  std::default_sentinel_t end() const noexcept { return std::default_sentinel; }
105
106  class iterator {
107   public:
108   //...
109    value_type operator*() const {
110      return parent_->value_;
111    }
112   private:
113    generate2* parent_;
114  };
115
116 private:
117   F generator_func_;
118   std::remove_cvref_t<std::invoke_result_t<F&>> value_; 
119};
120

1npm start
2> dataflow@0.1.0 start
3> react-scripts start
4
5node:internal/modules/cjs/loader:488
6      throw e;
7  ^
8
9Error [ERR_PACKAGE_PATH_NOT_EXPORTED]: Package subpath './lib/tokenize' is not defined by "exports" in /Users/juliantc/Desktop/ai-studio/development/frontend/node_modules/postcss-safe-parser/node_modules/postcss/package.json
10    at new NodeError (node:internal/errors:371:5)
11    at throwExportsNotFound (node:internal/modules/esm/resolve:416:9)
12    at packageExportsResolve (node:internal/modules/esm/resolve:669:3)
13    at resolveExports (node:internal/modules/cjs/loader:482:36)
14    at Function.Module._findPath (node:internal/modules/cjs/loader:522:31)
15    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:919:27)
16    at Function.Module._load (node:internal/modules/cjs/loader:778:27)
17    at Module.require (node:internal/modules/cjs/loader:999:19)
18    at require (node:internal/modules/cjs/helpers:102:18)
19    at Object.<anonymous> (/Users/juliantc/Desktop/ai-    studio/development/frontend/node_modules/postcss-safe-parser/lib/safe-parser.js:1:17) {
20  code: 'ERR_PACKAGE_PATH_NOT_EXPORTED'
21}
22
23Node.js v17.0.1
24{
25  "name": "postcss",
26  "version": "8.2.6",
27  "description": "Tool for transforming styles with JS plugins",
28  "engines": {
29    "node": "^10 || ^12 || >=14"
30  },
31  "exports": {
32    ".": {
33      "require": "./lib/postcss.js",
34      "import": "./lib/postcss.mjs",
35      "types": "./lib/postcss.d.ts"
36    },
37    "./": "./"
38  },
39  "main": "./lib/postcss.js",
40  "types": "./lib/postcss.d.ts",
41  "keywords": [
42    "css",
43    "postcss",
44    "rework",
45    "preprocessor",
46    "parser",
47    "source map",
48    "transform",
49    "manipulation",
50    "transpiler"
51  ],
52  "funding": {
53    "type": "opencollective",
54    "url": "https://opencollective.com/postcss/"
55  },
56  "author": "Andrey Sitnik <andrey@sitnik.ru>",
57  "license": "MIT",
58  "homepage": "https://postcss.org/",
59  "repository": "postcss/postcss",
60  "dependencies": {
61    "colorette": "^1.2.1",
62    "nanoid": "^3.1.20",
63    "source-map": "^0.6.1"
64  },
65  "browser": {
66    "./lib/terminal-highlight": false,
67    "colorette": false,
68    "fs": false
69  }
70}at-rule.d.ts        css-syntax-error.d.ts   input.d.ts      map-generator.js    postcss.d.ts        processor.js        rule.js         tokenize.js
71at-rule.js      css-syntax-error.js input.js        node.d.ts       postcss.js      result.d.ts     stringifier.js      warn-once.js
72comment.d.ts        declaration.d.ts    lazy-result.d.ts    node.js         postcss.mjs     result.js       stringify.d.ts      warning.d.ts
73comment.js      declaration.js      lazy-result.js      parse.d.ts      previous-map.d.ts   root.d.ts       stringify.js        warning.js
74container.d.ts      fromJSON.d.ts       list.d.ts       parse.js        previous-map.js     root.js         symbols.js
75container.js        fromJSON.js     list.js         parser.js       processor.d.ts      rule.d.ts       terminal-highlight.js
76nvm uninstall <version>
77

1npm start
2> dataflow@0.1.0 start
3> react-scripts start
4
5node:internal/modules/cjs/loader:488
6      throw e;
7  ^
8
9Error [ERR_PACKAGE_PATH_NOT_EXPORTED]: Package subpath './lib/tokenize' is not defined by "exports" in /Users/juliantc/Desktop/ai-studio/development/frontend/node_modules/postcss-safe-parser/node_modules/postcss/package.json
10    at new NodeError (node:internal/errors:371:5)
11    at throwExportsNotFound (node:internal/modules/esm/resolve:416:9)
12    at packageExportsResolve (node:internal/modules/esm/resolve:669:3)
13    at resolveExports (node:internal/modules/cjs/loader:482:36)
14    at Function.Module._findPath (node:internal/modules/cjs/loader:522:31)
15    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:919:27)
16    at Function.Module._load (node:internal/modules/cjs/loader:778:27)
17    at Module.require (node:internal/modules/cjs/loader:999:19)
18    at require (node:internal/modules/cjs/helpers:102:18)
19    at Object.<anonymous> (/Users/juliantc/Desktop/ai-    studio/development/frontend/node_modules/postcss-safe-parser/lib/safe-parser.js:1:17) {
20  code: 'ERR_PACKAGE_PATH_NOT_EXPORTED'
21}
22
23Node.js v17.0.1
24{
25  "name": "postcss",
26  "version": "8.2.6",
27  "description": "Tool for transforming styles with JS plugins",
28  "engines": {
29    "node": "^10 || ^12 || >=14"
30  },
31  "exports": {
32    ".": {
33      "require": "./lib/postcss.js",
34      "import": "./lib/postcss.mjs",
35      "types": "./lib/postcss.d.ts"
36    },
37    "./": "./"
38  },
39  "main": "./lib/postcss.js",
40  "types": "./lib/postcss.d.ts",
41  "keywords": [
42    "css",
43    "postcss",
44    "rework",
45    "preprocessor",
46    "parser",
47    "source map",
48    "transform",
49    "manipulation",
50    "transpiler"
51  ],
52  "funding": {
53    "type": "opencollective",
54    "url": "https://opencollective.com/postcss/"
55  },
56  "author": "Andrey Sitnik <andrey@sitnik.ru>",
57  "license": "MIT",
58  "homepage": "https://postcss.org/",
59  "repository": "postcss/postcss",
60  "dependencies": {
61    "colorette": "^1.2.1",
62    "nanoid": "^3.1.20",
63    "source-map": "^0.6.1"
64  },
65  "browser": {
66    "./lib/terminal-highlight": false,
67    "colorette": false,
68    "fs": false
69  }
70}at-rule.d.ts        css-syntax-error.d.ts   input.d.ts      map-generator.js    postcss.d.ts        processor.js        rule.js         tokenize.js
71at-rule.js      css-syntax-error.js input.js        node.d.ts       postcss.js      result.d.ts     stringifier.js      warn-once.js
72comment.d.ts        declaration.d.ts    lazy-result.d.ts    node.js         postcss.mjs     result.js       stringify.d.ts      warning.d.ts
73comment.js      declaration.js      lazy-result.js      parse.d.ts      previous-map.d.ts   root.d.ts       stringify.js        warning.js
74container.d.ts      fromJSON.d.ts       list.d.ts       parse.js        previous-map.js     root.js         symbols.js
75container.js        fromJSON.js     list.js         parser.js       processor.d.ts      rule.d.ts       terminal-highlight.js
76nvm uninstall <version>
77nvm uninstall v17.0.1
78

1npm start
2> dataflow@0.1.0 start
3> react-scripts start
4
5node:internal/modules/cjs/loader:488
6      throw e;
7  ^
8
9Error [ERR_PACKAGE_PATH_NOT_EXPORTED]: Package subpath './lib/tokenize' is not defined by "exports" in /Users/juliantc/Desktop/ai-studio/development/frontend/node_modules/postcss-safe-parser/node_modules/postcss/package.json
10    at new NodeError (node:internal/errors:371:5)
11    at throwExportsNotFound (node:internal/modules/esm/resolve:416:9)
12    at packageExportsResolve (node:internal/modules/esm/resolve:669:3)
13    at resolveExports (node:internal/modules/cjs/loader:482:36)
14    at Function.Module._findPath (node:internal/modules/cjs/loader:522:31)
15    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:919:27)
16    at Function.Module._load (node:internal/modules/cjs/loader:778:27)
17    at Module.require (node:internal/modules/cjs/loader:999:19)
18    at require (node:internal/modules/cjs/helpers:102:18)
19    at Object.<anonymous> (/Users/juliantc/Desktop/ai-    studio/development/frontend/node_modules/postcss-safe-parser/lib/safe-parser.js:1:17) {
20  code: 'ERR_PACKAGE_PATH_NOT_EXPORTED'
21}
22
23Node.js v17.0.1
24{
25  "name": "postcss",
26  "version": "8.2.6",
27  "description": "Tool for transforming styles with JS plugins",
28  "engines": {
29    "node": "^10 || ^12 || >=14"
30  },
31  "exports": {
32    ".": {
33      "require": "./lib/postcss.js",
34      "import": "./lib/postcss.mjs",
35      "types": "./lib/postcss.d.ts"
36    },
37    "./": "./"
38  },
39  "main": "./lib/postcss.js",
40  "types": "./lib/postcss.d.ts",
41  "keywords": [
42    "css",
43    "postcss",
44    "rework",
45    "preprocessor",
46    "parser",
47    "source map",
48    "transform",
49    "manipulation",
50    "transpiler"
51  ],
52  "funding": {
53    "type": "opencollective",
54    "url": "https://opencollective.com/postcss/"
55  },
56  "author": "Andrey Sitnik <andrey@sitnik.ru>",
57  "license": "MIT",
58  "homepage": "https://postcss.org/",
59  "repository": "postcss/postcss",
60  "dependencies": {
61    "colorette": "^1.2.1",
62    "nanoid": "^3.1.20",
63    "source-map": "^0.6.1"
64  },
65  "browser": {
66    "./lib/terminal-highlight": false,
67    "colorette": false,
68    "fs": false
69  }
70}at-rule.d.ts        css-syntax-error.d.ts   input.d.ts      map-generator.js    postcss.d.ts        processor.js        rule.js         tokenize.js
71at-rule.js      css-syntax-error.js input.js        node.d.ts       postcss.js      result.d.ts     stringifier.js      warn-once.js
72comment.d.ts        declaration.d.ts    lazy-result.d.ts    node.js         postcss.mjs     result.js       stringify.d.ts      warning.d.ts
73comment.js      declaration.js      lazy-result.js      parse.d.ts      previous-map.d.ts   root.d.ts       stringify.js        warning.js
74container.d.ts      fromJSON.d.ts       list.d.ts       parse.js        previous-map.js     root.js         symbols.js
75container.js        fromJSON.js     list.js         parser.js       processor.d.ts      rule.d.ts       terminal-highlight.js
76nvm uninstall <version>
77nvm uninstall v17.0.1
78nvm install --lts
79

1bytes_iter = (
2    b'col_1,',
3    b'c',
4    b'ol_2\n1',
5    b',"val',
6    b'ue"\n',
7)
8lines_iter = (
9    'col_1,col_2',
10    '1,"value"',
11)
12import re
13find_rule = re.compile("([^\r\n]*)(\r\n|\r|\n)?")
14
15
16def converter(byte_data):
17    left_d = ""
18    for d in byte_data:
19        # Used to save the previous match result in the `for` loop
20        prev_result = None
21        # Concatenate the last part of the previous data with the current data,
22        # used to deal with the case of `\r\n` being separated.
23        d = left_d + d.decode()
24        left_d = ""
25        # Using `find_rule.finditer` the last value("") will be invalid
26        for match_result in find_rule.finditer(d):
27            i = match_result.group()
28            if not i:
29                # The program comes to this point, indicating that i == "", which is the last matching value
30                left_d, prev_result = prev_result.group(), None
31                continue
32            if prev_result:
33                if prev_result.group(2) is None:
34                    # The program goes here, represented as the last valid value matched
35                    left_d = prev_result.group()
36                else:
37                    # Returns the previous matched value
38                    yield prev_result.group()
39            # Save the current match result
40            prev_result = match_result
41
42    else:
43        yield left_d
44
45
46for i in (converter(iter((
47        b'col_1,\r',
48        b'\nc',
49        b'ol_2\n1',
50        b'\n,"val;\r',
51        b'ue"\n')))
52):
53    print(repr(i))
54
55

1bytes_iter = (
2    b'col_1,',
3    b'c',
4    b'ol_2\n1',
5    b',"val',
6    b'ue"\n',
7)
8lines_iter = (
9    'col_1,col_2',
10    '1,"value"',
11)
12import re
13find_rule = re.compile("([^\r\n]*)(\r\n|\r|\n)?")
14
15
16def converter(byte_data):
17    left_d = ""
18    for d in byte_data:
19        # Used to save the previous match result in the `for` loop
20        prev_result = None
21        # Concatenate the last part of the previous data with the current data,
22        # used to deal with the case of `\r\n` being separated.
23        d = left_d + d.decode()
24        left_d = ""
25        # Using `find_rule.finditer` the last value("") will be invalid
26        for match_result in find_rule.finditer(d):
27            i = match_result.group()
28            if not i:
29                # The program comes to this point, indicating that i == "", which is the last matching value
30                left_d, prev_result = prev_result.group(), None
31                continue
32            if prev_result:
33                if prev_result.group(2) is None:
34                    # The program goes here, represented as the last valid value matched
35                    left_d = prev_result.group()
36                else:
37                    # Returns the previous matched value
38                    yield prev_result.group()
39            # Save the current match result
40            prev_result = match_result
41
42    else:
43        yield left_d
44
45
46for i in (converter(iter((
47        b'col_1,\r',
48        b'\nc',
49        b'ol_2\n1',
50        b'\n,"val;\r',
51        b'ue"\n')))
52):
53    print(repr(i))
54
55'col_1,\r\n'
56'col_2\n'
57'1\n'
58',"val;\r'
59'ue"\n'
60

1public static class StringCipher
2{
3    // This constant is used to determine the keysize of the encryption algorithm in bits.
4    // We divide this by 8 within the code below to get the equivalent number of bytes.
5    private const int Keysize = 128;
6
7    // This constant determines the number of iterations for the password bytes generation function.
8    private const int DerivationIterations = 1000;
9
10    public static string Encrypt(string plainText, string passPhrase)
11    {
12        // Salt and IV is randomly generated each time, but is preprended to encrypted cipher text
13        // so that the same Salt and IV values can be used when decrypting.  
14        var saltStringBytes = Generate128BitsOfRandomEntropy();
15        var ivStringBytes = Generate128BitsOfRandomEntropy();
16        var plainTextBytes = Encoding.UTF8.GetBytes(plainText);
17        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
18        {
19            var keyBytes = password.GetBytes(Keysize / 8);
20            using (var symmetricKey = Aes.Create())
21            {
22                symmetricKey.BlockSize = 128;
23                symmetricKey.Mode = CipherMode.CBC;
24                symmetricKey.Padding = PaddingMode.PKCS7;
25                using (var encryptor = symmetricKey.CreateEncryptor(keyBytes, ivStringBytes))
26                {
27                    using (var memoryStream = new MemoryStream())
28                    {
29                        using (var cryptoStream = new CryptoStream(memoryStream, encryptor, CryptoStreamMode.Write))
30                        {
31                            cryptoStream.Write(plainTextBytes, 0, plainTextBytes.Length);
32                            cryptoStream.FlushFinalBlock();
33                            // Create the final bytes as a concatenation of the random salt bytes, the random iv bytes and the cipher bytes.
34                            var cipherTextBytes = saltStringBytes;
35                            cipherTextBytes = cipherTextBytes.Concat(ivStringBytes).ToArray();
36                            cipherTextBytes = cipherTextBytes.Concat(memoryStream.ToArray()).ToArray();
37                            memoryStream.Close();
38                            cryptoStream.Close();
39                            return Convert.ToBase64String(cipherTextBytes);
40                        }
41                    }
42                }
43            }
44        }
45    }
46
47    public static string Decrypt(string cipherText, string passPhrase)
48    {
49        // Get the complete stream of bytes that represent:
50        // [32 bytes of Salt] + [16 bytes of IV] + [n bytes of CipherText]
51        var cipherTextBytesWithSaltAndIv = Convert.FromBase64String(cipherText);
52        // Get the saltbytes by extracting the first 16 bytes from the supplied cipherText bytes.
53        var saltStringBytes = cipherTextBytesWithSaltAndIv.Take(Keysize / 8).ToArray();
54        // Get the IV bytes by extracting the next 16 bytes from the supplied cipherText bytes.
55        var ivStringBytes = cipherTextBytesWithSaltAndIv.Skip(Keysize / 8).Take(Keysize / 8).ToArray();
56        // Get the actual cipher text bytes by removing the first 64 bytes from the cipherText string.
57        var cipherTextBytes = cipherTextBytesWithSaltAndIv.Skip((Keysize / 8) * 2).Take(cipherTextBytesWithSaltAndIv.Length - ((Keysize / 8) * 2)).ToArray();
58
59        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
60        {
61            var keyBytes = password.GetBytes(Keysize / 8);
62            using (var symmetricKey = Aes.Create())
63            {
64                symmetricKey.BlockSize = 128;
65                symmetricKey.Mode = CipherMode.CBC;
66                symmetricKey.Padding = PaddingMode.PKCS7;
67                using (var decryptor = symmetricKey.CreateDecryptor(keyBytes, ivStringBytes))
68                {
69                    using (var memoryStream = new MemoryStream(cipherTextBytes))
70                    {
71                        using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
72                        {
73                            var plainTextBytes = new byte[cipherTextBytes.Length];
74                            var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
75                            memoryStream.Close();
76                            cryptoStream.Close();
77                            return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
78                        }
79                    }
80                }
81            }
82        }
83    }
84
85    private static byte[] Generate128BitsOfRandomEntropy()
86    {
87        var randomBytes = new byte[16]; // 16 Bytes will give us 128 bits.
88        using (var rngCsp = RandomNumberGenerator.Create())
89        {
90            // Fill the array with cryptographically secure random bytes.
91            rngCsp.GetBytes(randomBytes);
92        }
93        return randomBytes;
94    }
95}
96var input = "12345678901234567890";
97var inputLength = input.Length;
98var inputBytes = Encoding.UTF8.GetBytes(input);
99
100var encrypted = StringCipher.Encrypt(input, "nzv86ri4H2qYHqc&m6rL");
101
102var output = StringCipher.Decrypt(encrypted, "nzv86ri4H2qYHqc&m6rL");
103var outputLength = output.Length;
104var outputBytes = Encoding.UTF8.GetBytes(output);
105
106var lengthDiff = inputLength - outputLength;
107using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
108{
109    var plainTextBytes = new byte[cipherTextBytes.Length];
110    var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
111    memoryStream.Close();
112    cryptoStream.Close();
113    return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
114}
115

1public static class StringCipher
2{
3    // This constant is used to determine the keysize of the encryption algorithm in bits.
4    // We divide this by 8 within the code below to get the equivalent number of bytes.
5    private const int Keysize = 128;
6
7    // This constant determines the number of iterations for the password bytes generation function.
8    private const int DerivationIterations = 1000;
9
10    public static string Encrypt(string plainText, string passPhrase)
11    {
12        // Salt and IV is randomly generated each time, but is preprended to encrypted cipher text
13        // so that the same Salt and IV values can be used when decrypting.  
14        var saltStringBytes = Generate128BitsOfRandomEntropy();
15        var ivStringBytes = Generate128BitsOfRandomEntropy();
16        var plainTextBytes = Encoding.UTF8.GetBytes(plainText);
17        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
18        {
19            var keyBytes = password.GetBytes(Keysize / 8);
20            using (var symmetricKey = Aes.Create())
21            {
22                symmetricKey.BlockSize = 128;
23                symmetricKey.Mode = CipherMode.CBC;
24                symmetricKey.Padding = PaddingMode.PKCS7;
25                using (var encryptor = symmetricKey.CreateEncryptor(keyBytes, ivStringBytes))
26                {
27                    using (var memoryStream = new MemoryStream())
28                    {
29                        using (var cryptoStream = new CryptoStream(memoryStream, encryptor, CryptoStreamMode.Write))
30                        {
31                            cryptoStream.Write(plainTextBytes, 0, plainTextBytes.Length);
32                            cryptoStream.FlushFinalBlock();
33                            // Create the final bytes as a concatenation of the random salt bytes, the random iv bytes and the cipher bytes.
34                            var cipherTextBytes = saltStringBytes;
35                            cipherTextBytes = cipherTextBytes.Concat(ivStringBytes).ToArray();
36                            cipherTextBytes = cipherTextBytes.Concat(memoryStream.ToArray()).ToArray();
37                            memoryStream.Close();
38                            cryptoStream.Close();
39                            return Convert.ToBase64String(cipherTextBytes);
40                        }
41                    }
42                }
43            }
44        }
45    }
46
47    public static string Decrypt(string cipherText, string passPhrase)
48    {
49        // Get the complete stream of bytes that represent:
50        // [32 bytes of Salt] + [16 bytes of IV] + [n bytes of CipherText]
51        var cipherTextBytesWithSaltAndIv = Convert.FromBase64String(cipherText);
52        // Get the saltbytes by extracting the first 16 bytes from the supplied cipherText bytes.
53        var saltStringBytes = cipherTextBytesWithSaltAndIv.Take(Keysize / 8).ToArray();
54        // Get the IV bytes by extracting the next 16 bytes from the supplied cipherText bytes.
55        var ivStringBytes = cipherTextBytesWithSaltAndIv.Skip(Keysize / 8).Take(Keysize / 8).ToArray();
56        // Get the actual cipher text bytes by removing the first 64 bytes from the cipherText string.
57        var cipherTextBytes = cipherTextBytesWithSaltAndIv.Skip((Keysize / 8) * 2).Take(cipherTextBytesWithSaltAndIv.Length - ((Keysize / 8) * 2)).ToArray();
58
59        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
60        {
61            var keyBytes = password.GetBytes(Keysize / 8);
62            using (var symmetricKey = Aes.Create())
63            {
64                symmetricKey.BlockSize = 128;
65                symmetricKey.Mode = CipherMode.CBC;
66                symmetricKey.Padding = PaddingMode.PKCS7;
67                using (var decryptor = symmetricKey.CreateDecryptor(keyBytes, ivStringBytes))
68                {
69                    using (var memoryStream = new MemoryStream(cipherTextBytes))
70                    {
71                        using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
72                        {
73                            var plainTextBytes = new byte[cipherTextBytes.Length];
74                            var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
75                            memoryStream.Close();
76                            cryptoStream.Close();
77                            return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
78                        }
79                    }
80                }
81            }
82        }
83    }
84
85    private static byte[] Generate128BitsOfRandomEntropy()
86    {
87        var randomBytes = new byte[16]; // 16 Bytes will give us 128 bits.
88        using (var rngCsp = RandomNumberGenerator.Create())
89        {
90            // Fill the array with cryptographically secure random bytes.
91            rngCsp.GetBytes(randomBytes);
92        }
93        return randomBytes;
94    }
95}
96var input = "12345678901234567890";
97var inputLength = input.Length;
98var inputBytes = Encoding.UTF8.GetBytes(input);
99
100var encrypted = StringCipher.Encrypt(input, "nzv86ri4H2qYHqc&m6rL");
101
102var output = StringCipher.Decrypt(encrypted, "nzv86ri4H2qYHqc&m6rL");
103var outputLength = output.Length;
104var outputBytes = Encoding.UTF8.GetBytes(output);
105
106var lengthDiff = inputLength - outputLength;
107using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
108{
109    var plainTextBytes = new byte[cipherTextBytes.Length];
110    var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
111    memoryStream.Close();
112    cryptoStream.Close();
113    return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
114}
115using (var plainTextStream = new MemoryStream())
116{
117    cryptoStream.CopyTo(plainTextStream);
118    var plainTextBytes = plainTextStream.ToArray();
119    return Encoding.UTF8.GetString(plainTextBytes, 0, plainTextBytes.Length);
120} 
121

1public static class StringCipher
2{
3    // This constant is used to determine the keysize of the encryption algorithm in bits.
4    // We divide this by 8 within the code below to get the equivalent number of bytes.
5    private const int Keysize = 128;
6
7    // This constant determines the number of iterations for the password bytes generation function.
8    private const int DerivationIterations = 1000;
9
10    public static string Encrypt(string plainText, string passPhrase)
11    {
12        // Salt and IV is randomly generated each time, but is preprended to encrypted cipher text
13        // so that the same Salt and IV values can be used when decrypting.  
14        var saltStringBytes = Generate128BitsOfRandomEntropy();
15        var ivStringBytes = Generate128BitsOfRandomEntropy();
16        var plainTextBytes = Encoding.UTF8.GetBytes(plainText);
17        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
18        {
19            var keyBytes = password.GetBytes(Keysize / 8);
20            using (var symmetricKey = Aes.Create())
21            {
22                symmetricKey.BlockSize = 128;
23                symmetricKey.Mode = CipherMode.CBC;
24                symmetricKey.Padding = PaddingMode.PKCS7;
25                using (var encryptor = symmetricKey.CreateEncryptor(keyBytes, ivStringBytes))
26                {
27                    using (var memoryStream = new MemoryStream())
28                    {
29                        using (var cryptoStream = new CryptoStream(memoryStream, encryptor, CryptoStreamMode.Write))
30                        {
31                            cryptoStream.Write(plainTextBytes, 0, plainTextBytes.Length);
32                            cryptoStream.FlushFinalBlock();
33                            // Create the final bytes as a concatenation of the random salt bytes, the random iv bytes and the cipher bytes.
34                            var cipherTextBytes = saltStringBytes;
35                            cipherTextBytes = cipherTextBytes.Concat(ivStringBytes).ToArray();
36                            cipherTextBytes = cipherTextBytes.Concat(memoryStream.ToArray()).ToArray();
37                            memoryStream.Close();
38                            cryptoStream.Close();
39                            return Convert.ToBase64String(cipherTextBytes);
40                        }
41                    }
42                }
43            }
44        }
45    }
46
47    public static string Decrypt(string cipherText, string passPhrase)
48    {
49        // Get the complete stream of bytes that represent:
50        // [32 bytes of Salt] + [16 bytes of IV] + [n bytes of CipherText]
51        var cipherTextBytesWithSaltAndIv = Convert.FromBase64String(cipherText);
52        // Get the saltbytes by extracting the first 16 bytes from the supplied cipherText bytes.
53        var saltStringBytes = cipherTextBytesWithSaltAndIv.Take(Keysize / 8).ToArray();
54        // Get the IV bytes by extracting the next 16 bytes from the supplied cipherText bytes.
55        var ivStringBytes = cipherTextBytesWithSaltAndIv.Skip(Keysize / 8).Take(Keysize / 8).ToArray();
56        // Get the actual cipher text bytes by removing the first 64 bytes from the cipherText string.
57        var cipherTextBytes = cipherTextBytesWithSaltAndIv.Skip((Keysize / 8) * 2).Take(cipherTextBytesWithSaltAndIv.Length - ((Keysize / 8) * 2)).ToArray();
58
59        using (var password = new Rfc2898DeriveBytes(passPhrase, saltStringBytes, DerivationIterations))
60        {
61            var keyBytes = password.GetBytes(Keysize / 8);
62            using (var symmetricKey = Aes.Create())
63            {
64                symmetricKey.BlockSize = 128;
65                symmetricKey.Mode = CipherMode.CBC;
66                symmetricKey.Padding = PaddingMode.PKCS7;
67                using (var decryptor = symmetricKey.CreateDecryptor(keyBytes, ivStringBytes))
68                {
69                    using (var memoryStream = new MemoryStream(cipherTextBytes))
70                    {
71                        using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
72                        {
73                            var plainTextBytes = new byte[cipherTextBytes.Length];
74                            var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
75                            memoryStream.Close();
76                            cryptoStream.Close();
77                            return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
78                        }
79                    }
80                }
81            }
82        }
83    }
84
85    private static byte[] Generate128BitsOfRandomEntropy()
86    {
87        var randomBytes = new byte[16]; // 16 Bytes will give us 128 bits.
88        using (var rngCsp = RandomNumberGenerator.Create())
89        {
90            // Fill the array with cryptographically secure random bytes.
91            rngCsp.GetBytes(randomBytes);
92        }
93        return randomBytes;
94    }
95}
96var input = "12345678901234567890";
97var inputLength = input.Length;
98var inputBytes = Encoding.UTF8.GetBytes(input);
99
100var encrypted = StringCipher.Encrypt(input, "nzv86ri4H2qYHqc&m6rL");
101
102var output = StringCipher.Decrypt(encrypted, "nzv86ri4H2qYHqc&m6rL");
103var outputLength = output.Length;
104var outputBytes = Encoding.UTF8.GetBytes(output);
105
106var lengthDiff = inputLength - outputLength;
107using (var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read))
108{
109    var plainTextBytes = new byte[cipherTextBytes.Length];
110    var decryptedByteCount = cryptoStream.Read(plainTextBytes, 0, plainTextBytes.Length);
111    memoryStream.Close();
112    cryptoStream.Close();
113    return Encoding.UTF8.GetString(plainTextBytes, 0, decryptedByteCount);
114}
115using (var plainTextStream = new MemoryStream())
116{
117    cryptoStream.CopyTo(plainTextStream);
118    var plainTextBytes = plainTextStream.ToArray();
119    return Encoding.UTF8.GetString(plainTextBytes, 0, plainTextBytes.Length);
120} 
121using (var plainTextReader = new StreamReader(cryptoStream))
122{
123    return plainTextReader.ReadToEnd();
124}  
125

1def print_twice(x):
2    for i in x: print(i)
3    for i in x: print(i)
4print_twice([1,2,3])
5print_twice((1,2,3))
6print_twice(zip([1,2,3],[4,5,6]))
7if isinstance(x, zip):
8  x = list(x)
9

1services.AddHttpClient<IClient, Client>((provider, client) =>
2    {
3        client.BaseAddress = new System.Uri("https://some.baseurl/");
4    });
5{"errorMessage":"Unable to resolve service for type 'System.String' while attempting to activate 'xxx.Client'."}
6public Client(string baseUrl, System.Net.Http.HttpClient httpClient)
7{
8    BaseUrl = baseUrl;
9    _httpClient = httpClient;
10    _settings = new System.Lazy<Newtonsoft.Json.JsonSerializerSettings>(CreateSerializerSettings);
11}
12  <ItemGroup>
13    <OpenApiReference Include="OpenAPIs\swagger.json" CodeGenerator="NSwagCSharp" Namespace="xxx" ClassName="Client">
14      <SourceUri>https://localhost:44353/swagger/v1/swagger.json</SourceUri>
15    </OpenApiReference>
16  </ItemGroup>
172>GenerateNSwagCSharp:
182>  "C:\.<path>./tools/Win/NSwag.exe" openapi2csclient /className:Client /namespace:xxx /input:"C:\<projpath>\OpenAPIs\swagger.json" /output:"obj\swaggerClient.cs"
192>NSwag command line tool for .NET 4.6.1+ WinX64, toolchain v13.13.2.0 (NJsonSchema v10.5.2.0 (Newtonsoft.Json v11.0.0.0))
20<ItemGroup>
21  <OpenApiReference Include="OpenAPIs\swagger.json" CodeGenerator="NSwagCSharp" Namespace="xxx" ClassName="Client">
22    <SourceUri>https://localhost:44353/swagger/v1/swagger.json</SourceUri>
23    <Options>/UseBaseUrl:false /GenerateClientInterfaces:true</Options>
24  </OpenApiReference>
25</ItemGroup>
26

1import unittest
2from typing import *
3
4T = TypeVar("T", bound=unittest.TestCase)
5
6def decorate(func: Callable[[T], None]) -> Callable[[T], None]:
7    def decorated_function(self: T) -> None:
8        return func(self)
9    return decorated_function
10my_decorate: Callable[[Callable[[T], None]], Callable[[T], None]] = decorate
11TD = Callable[[Callable[[T], None]], Callable[[T], None]]
12my_decorate: TD[T] = decorator_variable
13error: Type variable "mypytest.T" is unbound
14note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
15note: (Hint: Use "T" in function signature to bind "T" inside a function)
16error: Missing type parameters for generic type "TD"
17import unittest
18from typing import *
19
20T = TypeVar("T", bound=unittest.TestCase)
21
22
23def decorate(func: Callable[[T], None]) -> Callable[[T], None]:
24    def decorated_function(self: T) -> None:
25        return func(self)
26
27    return decorated_function
28
29
30decorator_variable: Callable[[Callable[[T], None]], Callable[[T], None]] = decorate
31
32U = Callable[[T], None]
33my_decorate: Callable[[U[T]], U[T]] = decorator_variable
34