Data Processing

1class Home extends StatelessWidget {
2  const Home({Key? key}) : super(key: key);
3
4  @override
5  Widget build(BuildContext context) {
6
7    return FutureBuilder(
8      future: Future.delayed(Duration(seconds: 3)),
9      builder: (BuildContext context, AsyncSnapshot snapshot){
10        if(snapshot.connectionState == ConnectionState.waiting)
11          return SplashUI();    ///Splash Screen
12        else
13          return MainUI();       ///Main Screen
14      },
15    );
16  }
17}
18
19Widget _splashUI(Size size){
20    return SafeArea(
21      child: Center(
22        child: Container(
23          width: size.width * 0.5,
24          height: size.height * 0.1,
25          child: Image(
26            fit: BoxFit.fill,
27            image: AssetImage('assets/images/elf_logo.png'),
28          ),
29        ),
30      ),
31    );
32  }
33
34 Widget build(BuildContext context) {
35
36 getFtpFile();
37 dataProgress();
38
39 return Platform.isAndroid ?
40    MaterialApp(
41      debugShowCheckedModeBanner: false,
42      home: Scaffold(
43        body: _splashUI(_size),
44      ),
45    ) :
46    CupertinoApp(
47      debugShowCheckedModeBanner: false,
48      home: CupertinoPageScaffold(
49        child: _splashUI(_size),
50      ),
51    );
52 }
53Future.wait([
54   getFTPFile(),
55   dataProgress(),     
56], () {
57  // once all Futures have completed, navigate to another page here
58});
59

1duration buf1: 8,890.551ms (8.9s)
2duration buf1: 8,339.755ms (8.3s)
3duration buf1: 8,620.633ms (8.6s)
4duration buf1: 8,682.809ms (8.7s)
5...
6make buffer 2 larger
7duration buf2 (resized): 19,542.750ms (19.5s)
8duration buf2 (resized): 22,423.529ms (22.4s)
9duration buf2 (resized): 22,413.364ms (22.4s)
10duration buf2 (resized): 22,219.383ms (22.2s)
11...
12import java.util.Locale;
13
14public final class Collector {
15
16    private static final boolean SELECT_QUICK = true;
17
18    private static final long LOOP_COUNT = 50_000L;
19    private static final int VALUE_COUNT = 150_000;
20    private static final int BUFFER_LENGTH = 100_000;
21
22    private final Buffer fBuffer = new Buffer();
23
24    public void reset() {fBuffer.reset();}
25    public void addValueBuf1(long val) {fBuffer.add1(val);}
26    public void addValueBuf2(long val) {fBuffer.add2(val);}
27
28    public static final class Buffer {
29
30        private int fIdx = 0;
31        private long[] fBuffer1 = new long[BUFFER_LENGTH * 2];
32        private long[] fBuffer2 = new long[BUFFER_LENGTH];
33
34        public void reset() {fIdx = 0;}
35
36        public void add1(long value) {
37            ensureRemainingBuf1(1);
38            fBuffer1[fIdx++] = value;
39        }
40
41        public void add2(long value) {
42            ensureRemainingBuf2(1);
43            fBuffer2[fIdx++] = value;
44        }
45
46        private void ensureRemainingBuf1(int remaining) {
47            if (remaining > fBuffer1.length - fIdx) {
48                System.out.println("make buffer 1 larger");
49                fBuffer1 = new long[(fIdx + remaining) << 1];
50            }
51        }
52
53        private void ensureRemainingBuf2(int remaining) {
54            if (remaining > fBuffer2.length - fIdx) {
55                System.out.println("make buffer 2 larger");
56                fBuffer2 = new long[(fIdx + remaining) << 1];
57            }
58        }
59
60    }
61
62    public static void main(String[] args) {
63        Locale.setDefault(Locale.ENGLISH);
64        final Collector collector = new Collector();
65        if (SELECT_QUICK) {
66            while (true) {
67                final long start = System.nanoTime();
68                for (long j = 0L; j < LOOP_COUNT; j++) {
69                    collector.reset();
70                    for (int k = 0; k < VALUE_COUNT; k++) {
71                        collector.addValueBuf1(k);
72                    }
73                }
74                final long nanos = System.nanoTime() - start;
75                System.out.printf("duration buf1: %1$,.3fms (%2$,.1fs)%n",
76                    nanos / 1_000_000d, nanos / 1_000_000_000d);
77            }
78        } else {
79            while (true) {
80                final long start = System.nanoTime();
81                for (long j = 0L; j < LOOP_COUNT; j++) {
82                    collector.reset();
83                    for (int k = 0; k < VALUE_COUNT; k++) {
84                        collector.addValueBuf2(k);
85                    }
86                }
87                final long nanos = System.nanoTime() - start;
88                System.out.printf("duration buf2 (resized): %1$,.3fms (%2$,.1fs)%n",
89                    nanos / 1_000_000d, nanos / 1_000_000_000d);
90            }
91        }
92    }
93
94}
95if (VALUE_COUNT > collector.fBuffer.fBuffer1.length) {
96    uncommon_trap();
97}
98for (int k = 0; k < VALUE_COUNT; k++) {
99    collector.fBuffer.fBuffer1[k] = k;  // no bounds check
100}
101

1duration buf1: 8,890.551ms (8.9s)
2duration buf1: 8,339.755ms (8.3s)
3duration buf1: 8,620.633ms (8.6s)
4duration buf1: 8,682.809ms (8.7s)
5...
6make buffer 2 larger
7duration buf2 (resized): 19,542.750ms (19.5s)
8duration buf2 (resized): 22,423.529ms (22.4s)
9duration buf2 (resized): 22,413.364ms (22.4s)
10duration buf2 (resized): 22,219.383ms (22.2s)
11...
12import java.util.Locale;
13
14public final class Collector {
15
16    private static final boolean SELECT_QUICK = true;
17
18    private static final long LOOP_COUNT = 50_000L;
19    private static final int VALUE_COUNT = 150_000;
20    private static final int BUFFER_LENGTH = 100_000;
21
22    private final Buffer fBuffer = new Buffer();
23
24    public void reset() {fBuffer.reset();}
25    public void addValueBuf1(long val) {fBuffer.add1(val);}
26    public void addValueBuf2(long val) {fBuffer.add2(val);}
27
28    public static final class Buffer {
29
30        private int fIdx = 0;
31        private long[] fBuffer1 = new long[BUFFER_LENGTH * 2];
32        private long[] fBuffer2 = new long[BUFFER_LENGTH];
33
34        public void reset() {fIdx = 0;}
35
36        public void add1(long value) {
37            ensureRemainingBuf1(1);
38            fBuffer1[fIdx++] = value;
39        }
40
41        public void add2(long value) {
42            ensureRemainingBuf2(1);
43            fBuffer2[fIdx++] = value;
44        }
45
46        private void ensureRemainingBuf1(int remaining) {
47            if (remaining > fBuffer1.length - fIdx) {
48                System.out.println("make buffer 1 larger");
49                fBuffer1 = new long[(fIdx + remaining) << 1];
50            }
51        }
52
53        private void ensureRemainingBuf2(int remaining) {
54            if (remaining > fBuffer2.length - fIdx) {
55                System.out.println("make buffer 2 larger");
56                fBuffer2 = new long[(fIdx + remaining) << 1];
57            }
58        }
59
60    }
61
62    public static void main(String[] args) {
63        Locale.setDefault(Locale.ENGLISH);
64        final Collector collector = new Collector();
65        if (SELECT_QUICK) {
66            while (true) {
67                final long start = System.nanoTime();
68                for (long j = 0L; j < LOOP_COUNT; j++) {
69                    collector.reset();
70                    for (int k = 0; k < VALUE_COUNT; k++) {
71                        collector.addValueBuf1(k);
72                    }
73                }
74                final long nanos = System.nanoTime() - start;
75                System.out.printf("duration buf1: %1$,.3fms (%2$,.1fs)%n",
76                    nanos / 1_000_000d, nanos / 1_000_000_000d);
77            }
78        } else {
79            while (true) {
80                final long start = System.nanoTime();
81                for (long j = 0L; j < LOOP_COUNT; j++) {
82                    collector.reset();
83                    for (int k = 0; k < VALUE_COUNT; k++) {
84                        collector.addValueBuf2(k);
85                    }
86                }
87                final long nanos = System.nanoTime() - start;
88                System.out.printf("duration buf2 (resized): %1$,.3fms (%2$,.1fs)%n",
89                    nanos / 1_000_000d, nanos / 1_000_000_000d);
90            }
91        }
92    }
93
94}
95if (VALUE_COUNT > collector.fBuffer.fBuffer1.length) {
96    uncommon_trap();
97}
98for (int k = 0; k < VALUE_COUNT; k++) {
99    collector.fBuffer.fBuffer1[k] = k;  // no bounds check
100}
101for (long j = 0L; j < LOOP_COUNT; j++) {
102    collector.reset();
103
104    int fastCount = Math.min(collector.fBuffer.fBuffer2.length, VALUE_COUNT);
105    for (int k = 0; k < fastCount; k++) {
106        collector.addValueBuf2Fast(k);
107    }
108
109    for (int k = fastCount; k < VALUE_COUNT; k++) {
110        collector.addValueBuf2(k);
111    }
112}
113

1duration buf1: 8,890.551ms (8.9s)
2duration buf1: 8,339.755ms (8.3s)
3duration buf1: 8,620.633ms (8.6s)
4duration buf1: 8,682.809ms (8.7s)
5...
6make buffer 2 larger
7duration buf2 (resized): 19,542.750ms (19.5s)
8duration buf2 (resized): 22,423.529ms (22.4s)
9duration buf2 (resized): 22,413.364ms (22.4s)
10duration buf2 (resized): 22,219.383ms (22.2s)
11...
12import java.util.Locale;
13
14public final class Collector {
15
16    private static final boolean SELECT_QUICK = true;
17
18    private static final long LOOP_COUNT = 50_000L;
19    private static final int VALUE_COUNT = 150_000;
20    private static final int BUFFER_LENGTH = 100_000;
21
22    private final Buffer fBuffer = new Buffer();
23
24    public void reset() {fBuffer.reset();}
25    public void addValueBuf1(long val) {fBuffer.add1(val);}
26    public void addValueBuf2(long val) {fBuffer.add2(val);}
27
28    public static final class Buffer {
29
30        private int fIdx = 0;
31        private long[] fBuffer1 = new long[BUFFER_LENGTH * 2];
32        private long[] fBuffer2 = new long[BUFFER_LENGTH];
33
34        public void reset() {fIdx = 0;}
35
36        public void add1(long value) {
37            ensureRemainingBuf1(1);
38            fBuffer1[fIdx++] = value;
39        }
40
41        public void add2(long value) {
42            ensureRemainingBuf2(1);
43            fBuffer2[fIdx++] = value;
44        }
45
46        private void ensureRemainingBuf1(int remaining) {
47            if (remaining > fBuffer1.length - fIdx) {
48                System.out.println("make buffer 1 larger");
49                fBuffer1 = new long[(fIdx + remaining) << 1];
50            }
51        }
52
53        private void ensureRemainingBuf2(int remaining) {
54            if (remaining > fBuffer2.length - fIdx) {
55                System.out.println("make buffer 2 larger");
56                fBuffer2 = new long[(fIdx + remaining) << 1];
57            }
58        }
59
60    }
61
62    public static void main(String[] args) {
63        Locale.setDefault(Locale.ENGLISH);
64        final Collector collector = new Collector();
65        if (SELECT_QUICK) {
66            while (true) {
67                final long start = System.nanoTime();
68                for (long j = 0L; j < LOOP_COUNT; j++) {
69                    collector.reset();
70                    for (int k = 0; k < VALUE_COUNT; k++) {
71                        collector.addValueBuf1(k);
72                    }
73                }
74                final long nanos = System.nanoTime() - start;
75                System.out.printf("duration buf1: %1$,.3fms (%2$,.1fs)%n",
76                    nanos / 1_000_000d, nanos / 1_000_000_000d);
77            }
78        } else {
79            while (true) {
80                final long start = System.nanoTime();
81                for (long j = 0L; j < LOOP_COUNT; j++) {
82                    collector.reset();
83                    for (int k = 0; k < VALUE_COUNT; k++) {
84                        collector.addValueBuf2(k);
85                    }
86                }
87                final long nanos = System.nanoTime() - start;
88                System.out.printf("duration buf2 (resized): %1$,.3fms (%2$,.1fs)%n",
89                    nanos / 1_000_000d, nanos / 1_000_000_000d);
90            }
91        }
92    }
93
94}
95if (VALUE_COUNT > collector.fBuffer.fBuffer1.length) {
96    uncommon_trap();
97}
98for (int k = 0; k < VALUE_COUNT; k++) {
99    collector.fBuffer.fBuffer1[k] = k;  // no bounds check
100}
101for (long j = 0L; j < LOOP_COUNT; j++) {
102    collector.reset();
103
104    int fastCount = Math.min(collector.fBuffer.fBuffer2.length, VALUE_COUNT);
105    for (int k = 0; k < fastCount; k++) {
106        collector.addValueBuf2Fast(k);
107    }
108
109    for (int k = fastCount; k < VALUE_COUNT; k++) {
110        collector.addValueBuf2(k);
111    }
112}
113    public void addValueBuf2Fast(long val) {fBuffer.add2Fast(val);}
114
115    public static final class Buffer {
116        ...
117        public void add2Fast(long value) {
118            fBuffer2[fIdx++] = value;
119        }
120    }
121

1duration buf1: 8,890.551ms (8.9s)
2duration buf1: 8,339.755ms (8.3s)
3duration buf1: 8,620.633ms (8.6s)
4duration buf1: 8,682.809ms (8.7s)
5...
6make buffer 2 larger
7duration buf2 (resized): 19,542.750ms (19.5s)
8duration buf2 (resized): 22,423.529ms (22.4s)
9duration buf2 (resized): 22,413.364ms (22.4s)
10duration buf2 (resized): 22,219.383ms (22.2s)
11...
12import java.util.Locale;
13
14public final class Collector {
15
16    private static final boolean SELECT_QUICK = true;
17
18    private static final long LOOP_COUNT = 50_000L;
19    private static final int VALUE_COUNT = 150_000;
20    private static final int BUFFER_LENGTH = 100_000;
21
22    private final Buffer fBuffer = new Buffer();
23
24    public void reset() {fBuffer.reset();}
25    public void addValueBuf1(long val) {fBuffer.add1(val);}
26    public void addValueBuf2(long val) {fBuffer.add2(val);}
27
28    public static final class Buffer {
29
30        private int fIdx = 0;
31        private long[] fBuffer1 = new long[BUFFER_LENGTH * 2];
32        private long[] fBuffer2 = new long[BUFFER_LENGTH];
33
34        public void reset() {fIdx = 0;}
35
36        public void add1(long value) {
37            ensureRemainingBuf1(1);
38            fBuffer1[fIdx++] = value;
39        }
40
41        public void add2(long value) {
42            ensureRemainingBuf2(1);
43            fBuffer2[fIdx++] = value;
44        }
45
46        private void ensureRemainingBuf1(int remaining) {
47            if (remaining > fBuffer1.length - fIdx) {
48                System.out.println("make buffer 1 larger");
49                fBuffer1 = new long[(fIdx + remaining) << 1];
50            }
51        }
52
53        private void ensureRemainingBuf2(int remaining) {
54            if (remaining > fBuffer2.length - fIdx) {
55                System.out.println("make buffer 2 larger");
56                fBuffer2 = new long[(fIdx + remaining) << 1];
57            }
58        }
59
60    }
61
62    public static void main(String[] args) {
63        Locale.setDefault(Locale.ENGLISH);
64        final Collector collector = new Collector();
65        if (SELECT_QUICK) {
66            while (true) {
67                final long start = System.nanoTime();
68                for (long j = 0L; j < LOOP_COUNT; j++) {
69                    collector.reset();
70                    for (int k = 0; k < VALUE_COUNT; k++) {
71                        collector.addValueBuf1(k);
72                    }
73                }
74                final long nanos = System.nanoTime() - start;
75                System.out.printf("duration buf1: %1$,.3fms (%2$,.1fs)%n",
76                    nanos / 1_000_000d, nanos / 1_000_000_000d);
77            }
78        } else {
79            while (true) {
80                final long start = System.nanoTime();
81                for (long j = 0L; j < LOOP_COUNT; j++) {
82                    collector.reset();
83                    for (int k = 0; k < VALUE_COUNT; k++) {
84                        collector.addValueBuf2(k);
85                    }
86                }
87                final long nanos = System.nanoTime() - start;
88                System.out.printf("duration buf2 (resized): %1$,.3fms (%2$,.1fs)%n",
89                    nanos / 1_000_000d, nanos / 1_000_000_000d);
90            }
91        }
92    }
93
94}
95if (VALUE_COUNT > collector.fBuffer.fBuffer1.length) {
96    uncommon_trap();
97}
98for (int k = 0; k < VALUE_COUNT; k++) {
99    collector.fBuffer.fBuffer1[k] = k;  // no bounds check
100}
101for (long j = 0L; j < LOOP_COUNT; j++) {
102    collector.reset();
103
104    int fastCount = Math.min(collector.fBuffer.fBuffer2.length, VALUE_COUNT);
105    for (int k = 0; k < fastCount; k++) {
106        collector.addValueBuf2Fast(k);
107    }
108
109    for (int k = fastCount; k < VALUE_COUNT; k++) {
110        collector.addValueBuf2(k);
111    }
112}
113    public void addValueBuf2Fast(long val) {fBuffer.add2Fast(val);}
114
115    public static final class Buffer {
116        ...
117        public void add2Fast(long value) {
118            fBuffer2[fIdx++] = value;
119        }
120    }
121make buffer 2 larger
122duration buf2 (resized): 5,537.681ms (5.5s)
123duration buf2 (resized): 5,461.519ms (5.5s)
124duration buf2 (resized): 5,450.445ms (5.5s)
125

1# imports
2import pandas as pd  # data processing
3import numpy as np  # linear algebra
4import matplotlib.pyplot as plt  # plotting
5from datetime import date  # date
6from nsepy import get_history  # NSE historical data
7from keras.models import Sequential  # neural network
8from keras.layers import LSTM, Dropout, Dense  # LSTM layer
9from sklearn.preprocessing import MinMaxScaler  # scaling
10
11nseCode = 'TCS'
12stockTitle = 'Tata Consultancy Services'
13
14# API call
15apiData = get_history(symbol = nseCode, start = date(2017,1,1), end = date(2021,12,19))
16data = apiData  # copy the dataframe (not necessary)
17
18# remove columns you don't need
19del data['Symbol']
20del data['Series']
21del data['Prev Close']
22del data['Volume']
23del data['Turnover']
24del data['Trades']
25del data['Deliverable Volume']
26del data['%Deliverble']
27
28# store the data in a csv file
29data.to_csv('infy2.csv')
30
31# Read the csv file
32data = pd.read_csv('infy2.csv')
33
34# convert the date column to datetime; if you read data from csv, do this. Otherwise, no need if you read data from API
35data['Date'] = pd.to_datetime(data['Date'], format = '%Y-%m-%d')
36data.index = data['Date']
37
38# plot
39plt.xlabel('Date')
40plt.ylabel('Close Price (Rs.)')
41data['Close'].plot(legend = True, figsize = (10,6), title = stockTitle, grid = True, color = 'blue')
42
43# Sort data into Date and Close columns
44data2 = data.sort_index(ascending = True, axis = 0)
45
46newData = pd.DataFrame(index = range(0,len(data2)), columns = ['Date', 'Close'])
47
48for i in range(0, len(data2)):  # only if you read data from csv
49    newData['Date'][i] = data2['Date'][i]
50    newData['Close'][i] = data2['Close'][I]
51
52# Calculate the row number to split the dataset into train and test
53split = len(newData) - 100
54
55# normalize the new dataset
56scaler = MinMaxScaler(feature_range = (0, 1))
57finalData = newData.values
58
59trainData = finalData[0:split, :]
60validData = finalData[split:, :]
61
62newData.index = newData.Date
63newData.drop('Date', axis = 1, inplace = True)
64scaler = MinMaxScaler(feature_range = (0, 1))
65scaledData = scaler.fit_transform(newData)
66
67xTrainData, yTrainData = [], []
68
69for i in range(60, len(trainData)):  # data-flair has used 60 instead of 30
70    xTrainData.append(scaledData[i-60:i, 0])
71    yTrainData.append(scaledData[i, 0])
72
73xTrainData, yTrainData = np.array(xTrainData), np.array(yTrainData)
74
75xTrainData = np.reshape(xTrainData, (xTrainData.shape[0], xTrainData.shape[1], 1))
76
77# build and train the LSTM model
78lstmModel = Sequential()
79lstmModel.add(LSTM(units = 50, return_sequences = True, input_shape = (xTrainData.shape[1], 1)))
80lstmModel.add(LSTM(units = 50))
81lstmModel.add(Dense(units = 1))
82
83inputsData = newData[len(newData) - len(validData) - 60:].values
84inputsData = inputsData.reshape(-1,1)
85inputsData = scaler.transform(inputsData)
86
87lstmModel.compile(loss = 'mean_squared_error', optimizer = 'adam')
88lstmModel.fit(xTrainData, yTrainData, epochs = 1, batch_size = 1, verbose = 2)
89
90# Take a sample of a dataset to make predictions
91xTestData = []
92
93for i in range(60, inputsData.shape[0]):
94    xTestData.append(inputsData[i-60:i, 0])
95
96xTestData = np.array(xTestData)
97
98xTestData = np.reshape(xTestData, (xTestData.shape[0], xTestData.shape[1], 1))
99
100predictedClosingPrice = lstmModel.predict(xTestData)
101predictedClosingPrice = scaler.inverse_transform(predictedClosingPrice)
102
103# visualize the results
104trainData = newData[:split]
105validData = newData[split:]
106
107validData['Predictions'] = predictedClosingPrice
108
109plt.xlabel('Date')
110plt.ylabel('Close Price (Rs.)')
111
112trainData['Close'].plot(legend = True, color = 'blue', label = 'Train Data')
113validData['Close'].plot(legend = True, color = 'green', label = 'Valid Data')
114validData['Predictions'].plot(legend = True, figsize = (12,7), grid = True, color = 'orange', label = 'Predicted Data', title = stockTitle)
115import pandas as pd
116import numpy as np
117from datetime import date
118from nsepy import get_history
119from keras.models import Sequential
120from keras.layers import LSTM, Dense
121from sklearn.preprocessing import MinMaxScaler
122pd.options.mode.chained_assignment = None
123
124# load the data
125stock_ticker = 'TCS'
126stock_name = 'Tata Consultancy Services'
127train_start = date(2017, 1, 1)
128train_end = date.today()
129data = get_history(symbol=stock_ticker, start=train_start, end=train_end)
130data.index = pd.DatetimeIndex(data.index)
131data = data[['Close']]
132
133# scale the data
134scaler = MinMaxScaler(feature_range=(0, 1)).fit(data)
135z = scaler.transform(data)
136
137# extract the input sequences and target values
138window_size = 60
139
140x, y = [], []
141
142for i in range(window_size, len(z)):
143    x.append(z[i - window_size: i])
144    y.append(z[i])
145
146x, y = np.array(x), np.array(y)
147
148# build and train the model
149model = Sequential()
150model.add(LSTM(units=50, return_sequences=True, input_shape=x.shape[1:]))
151model.add(LSTM(units=50))
152model.add(Dense(units=1))
153model.compile(loss='mse', optimizer='adam')
154model.fit(x, y, epochs=100, batch_size=128, verbose=1)
155
156# generate the multi-step forecasts
157def multi_step_forecasts(n_past, n_future):
158
159    x_past = x[- n_past - 1:, :, :][:1]  # last observed input sequence
160    y_past = y[- n_past - 1]             # last observed target value
161    y_future = []                        # predicted target values
162
163    for i in range(n_past + n_future):
164
165        # feed the last forecast back to the model as an input
166        x_past = np.append(x_past[:, 1:, :], y_past.reshape(1, 1, 1), axis=1)
167
168        # generate the next forecast
169        y_past = model.predict(x_past)
170
171        # save the forecast
172        y_future.append(y_past.flatten()[0])
173
174    # transform the forecasts back to the original scale
175    y_future = scaler.inverse_transform(np.array(y_future).reshape(-1, 1)).flatten()
176
177    # add the forecasts to the data frame
178    df_past = data.rename(columns={'Close': 'Actual'}).copy()
179
180    df_future = pd.DataFrame(
181        index=pd.bdate_range(start=data.index[- n_past - 1] + pd.Timedelta(days=1), periods=n_past + n_future),
182        columns=['Forecast'],
183        data=y_future
184    )
185
186    return df_past.join(df_future, how='outer')
187
188# forecast the next 30 days
189df1 = multi_step_forecasts(n_past=0, n_future=30)
190df1.plot(title=stock_name)
191
192# forecast the last 20 days and the next 30 days
193df2 = multi_step_forecasts(n_past=20, n_future=30)
194df2.plot(title=stock_name)
195

1import numpy as np
2import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
3import os
4import scipy.io   
5from sklearn.neighbors import KNeighborsClassifier
6from sklearn import metrics
7from sklearn.model_selection import train_test_split
8from sklearn.preprocessing import StandardScaler
9from torch.utils.data import Dataset, DataLoader
10from sklearn import preprocessing
11import torch
12import numpy as np
13from sklearn.model_selection import KFold
14from sklearn.neighbors import KNeighborsClassifier
15from sklearn import metrics
16
17def load_mat_data(path):
18    mat = scipy.io.loadmat(DATA_PATH)
19    x,y = mat['data'], mat['class']
20    x = x.astype('float32')
21    # stadardize values
22    standardizer = preprocessing.StandardScaler()
23    x = standardizer.fit_transform(x) 
24    return x, standardizer, y
25
26def numpyToTensor(x):
27    x_train = torch.from_numpy(x)
28    return x_train
29
30class DataBuilder(Dataset):
31    def __init__(self, path):
32        self.x, self.standardizer, self.y = load_mat_data(DATA_PATH)
33        self.x = numpyToTensor(self.x)
34        self.len=self.x.shape[0]
35        self.y = numpyToTensor(self.y)
36    def __getitem__(self,index):      
37        return (self.x[index], self.y[index])
38    def __len__(self):
39        return self.len
40
41datasets = ['/home/katerina/Desktop/datasets/GSE75110.mat']
42
43for DATA_PATH in datasets:
44
45    print(DATA_PATH)
46    data_set=DataBuilder(DATA_PATH)
47
48    pred_rpknn = [0] * len(data_set.y)
49    kf = KFold(n_splits=10, shuffle = True, random_state=7)
50
51    for train_index, test_index in kf.split(data_set.x):
52        #Create KNN Classifier
53        knn = KNeighborsClassifier(n_neighbors=5)
54        #print("TRAIN:", train_index, "TEST:", test_index)
55        x_train, x_test = data_set.x[train_index], data_set.x[test_index]
56        y_train, y_test = data_set.y[train_index], data_set.y[test_index]
57        #Train the model using the training sets
58        y1_train = y_train.ravel()
59        knn.fit(x_train, y1_train)
60        #Predict the response for test dataset
61        y_pred = knn.predict(x_test)
62        #print(y_pred)
63        # Model Accuracy, how often is the classifier correct?
64        print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
65        c = 0
66        for idx in test_index:
67            pred_rpknn[idx] = y_pred[c]
68            c +=1
69    print("Accuracy:",metrics.accuracy_score(data_set.y, pred_rpknn))
70    print(pred_rpknn, data_set.y.reshape(1,-1))
71/home/katerina/Desktop/datasets/GSE75110.mat
72Accuracy: 0.2857142857142857
73Accuracy: 0.38095238095238093
74Accuracy: 0.14285714285714285
75Accuracy: 0.4
76Accuracy: 0.3
77Accuracy: 0.25
78Accuracy: 0.3
79Accuracy: 0.6
80Accuracy: 0.25
81Accuracy: 0.45
82Accuracy: 0.33497536945812806
83[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
84from sklearn import preprocessing
85
86...
87
88x = preprocessing.normalize(x)
89

1import numpy as np
2import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
3import os
4import scipy.io   
5from sklearn.neighbors import KNeighborsClassifier
6from sklearn import metrics
7from sklearn.model_selection import train_test_split
8from sklearn.preprocessing import StandardScaler
9from torch.utils.data import Dataset, DataLoader
10from sklearn import preprocessing
11import torch
12import numpy as np
13from sklearn.model_selection import KFold
14from sklearn.neighbors import KNeighborsClassifier
15from sklearn import metrics
16
17def load_mat_data(path):
18    mat = scipy.io.loadmat(DATA_PATH)
19    x,y = mat['data'], mat['class']
20    x = x.astype('float32')
21    # stadardize values
22    standardizer = preprocessing.StandardScaler()
23    x = standardizer.fit_transform(x) 
24    return x, standardizer, y
25
26def numpyToTensor(x):
27    x_train = torch.from_numpy(x)
28    return x_train
29
30class DataBuilder(Dataset):
31    def __init__(self, path):
32        self.x, self.standardizer, self.y = load_mat_data(DATA_PATH)
33        self.x = numpyToTensor(self.x)
34        self.len=self.x.shape[0]
35        self.y = numpyToTensor(self.y)
36    def __getitem__(self,index):      
37        return (self.x[index], self.y[index])
38    def __len__(self):
39        return self.len
40
41datasets = ['/home/katerina/Desktop/datasets/GSE75110.mat']
42
43for DATA_PATH in datasets:
44
45    print(DATA_PATH)
46    data_set=DataBuilder(DATA_PATH)
47
48    pred_rpknn = [0] * len(data_set.y)
49    kf = KFold(n_splits=10, shuffle = True, random_state=7)
50
51    for train_index, test_index in kf.split(data_set.x):
52        #Create KNN Classifier
53        knn = KNeighborsClassifier(n_neighbors=5)
54        #print("TRAIN:", train_index, "TEST:", test_index)
55        x_train, x_test = data_set.x[train_index], data_set.x[test_index]
56        y_train, y_test = data_set.y[train_index], data_set.y[test_index]
57        #Train the model using the training sets
58        y1_train = y_train.ravel()
59        knn.fit(x_train, y1_train)
60        #Predict the response for test dataset
61        y_pred = knn.predict(x_test)
62        #print(y_pred)
63        # Model Accuracy, how often is the classifier correct?
64        print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
65        c = 0
66        for idx in test_index:
67            pred_rpknn[idx] = y_pred[c]
68            c +=1
69    print("Accuracy:",metrics.accuracy_score(data_set.y, pred_rpknn))
70    print(pred_rpknn, data_set.y.reshape(1,-1))
71/home/katerina/Desktop/datasets/GSE75110.mat
72Accuracy: 0.2857142857142857
73Accuracy: 0.38095238095238093
74Accuracy: 0.14285714285714285
75Accuracy: 0.4
76Accuracy: 0.3
77Accuracy: 0.25
78Accuracy: 0.3
79Accuracy: 0.6
80Accuracy: 0.25
81Accuracy: 0.45
82Accuracy: 0.33497536945812806
83[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
84from sklearn import preprocessing
85
86...
87
88x = preprocessing.normalize(x)
89The standard score of a sample `x` is calculated as:
90
91    z = (x - u) / s
92
93where `u` is the mean of the training samples or zero if `with_mean=False`,
94and `s` is the standard deviation of the training samples or one if
95`with_std=False`.
96

1import numpy as np
2import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
3import os
4import scipy.io   
5from sklearn.neighbors import KNeighborsClassifier
6from sklearn import metrics
7from sklearn.model_selection import train_test_split
8from sklearn.preprocessing import StandardScaler
9from torch.utils.data import Dataset, DataLoader
10from sklearn import preprocessing
11import torch
12import numpy as np
13from sklearn.model_selection import KFold
14from sklearn.neighbors import KNeighborsClassifier
15from sklearn import metrics
16
17def load_mat_data(path):
18    mat = scipy.io.loadmat(DATA_PATH)
19    x,y = mat['data'], mat['class']
20    x = x.astype('float32')
21    # stadardize values
22    standardizer = preprocessing.StandardScaler()
23    x = standardizer.fit_transform(x) 
24    return x, standardizer, y
25
26def numpyToTensor(x):
27    x_train = torch.from_numpy(x)
28    return x_train
29
30class DataBuilder(Dataset):
31    def __init__(self, path):
32        self.x, self.standardizer, self.y = load_mat_data(DATA_PATH)
33        self.x = numpyToTensor(self.x)
34        self.len=self.x.shape[0]
35        self.y = numpyToTensor(self.y)
36    def __getitem__(self,index):      
37        return (self.x[index], self.y[index])
38    def __len__(self):
39        return self.len
40
41datasets = ['/home/katerina/Desktop/datasets/GSE75110.mat']
42
43for DATA_PATH in datasets:
44
45    print(DATA_PATH)
46    data_set=DataBuilder(DATA_PATH)
47
48    pred_rpknn = [0] * len(data_set.y)
49    kf = KFold(n_splits=10, shuffle = True, random_state=7)
50
51    for train_index, test_index in kf.split(data_set.x):
52        #Create KNN Classifier
53        knn = KNeighborsClassifier(n_neighbors=5)
54        #print("TRAIN:", train_index, "TEST:", test_index)
55        x_train, x_test = data_set.x[train_index], data_set.x[test_index]
56        y_train, y_test = data_set.y[train_index], data_set.y[test_index]
57        #Train the model using the training sets
58        y1_train = y_train.ravel()
59        knn.fit(x_train, y1_train)
60        #Predict the response for test dataset
61        y_pred = knn.predict(x_test)
62        #print(y_pred)
63        # Model Accuracy, how often is the classifier correct?
64        print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
65        c = 0
66        for idx in test_index:
67            pred_rpknn[idx] = y_pred[c]
68            c +=1
69    print("Accuracy:",metrics.accuracy_score(data_set.y, pred_rpknn))
70    print(pred_rpknn, data_set.y.reshape(1,-1))
71/home/katerina/Desktop/datasets/GSE75110.mat
72Accuracy: 0.2857142857142857
73Accuracy: 0.38095238095238093
74Accuracy: 0.14285714285714285
75Accuracy: 0.4
76Accuracy: 0.3
77Accuracy: 0.25
78Accuracy: 0.3
79Accuracy: 0.6
80Accuracy: 0.25
81Accuracy: 0.45
82Accuracy: 0.33497536945812806
83[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]
84from sklearn import preprocessing
85
86...
87
88x = preprocessing.normalize(x)
89The standard score of a sample `x` is calculated as:
90
91    z = (x - u) / s
92
93where `u` is the mean of the training samples or zero if `with_mean=False`,
94and `s` is the standard deviation of the training samples or one if
95`with_std=False`.
96import scipy.io
97from torch.utils.data import Dataset
98from sklearn import preprocessing
99import torch
100import numpy as np
101from sklearn.model_selection import KFold
102from sklearn.neighbors import KNeighborsClassifier
103from sklearn import metrics
104
105def load_mat_data(path):
106    mat = scipy.io.loadmat(DATA_PATH)
107    x, y = mat['data'], mat['class']
108    x = x.astype('float32')
109    # stadardize values
110    x = preprocessing.normalize(x)
111    return x, y
112
113def numpyToTensor(x):
114    x_train = torch.from_numpy(x)
115    return x_train
116
117class DataBuilder(Dataset):
118    def __init__(self, path):
119        self.x, self.y = load_mat_data(DATA_PATH)
120        self.x = numpyToTensor(self.x)
121        self.len=self.x.shape[0]
122        self.y = numpyToTensor(self.y)
123    def __getitem__(self,index):
124        return (self.x[index], self.y[index])
125    def __len__(self):
126        return self.len
127
128datasets = ['/home/katerina/Desktop/datasets/GSE75110.mat']
129
130for DATA_PATH in datasets:
131
132    print(DATA_PATH)
133    data_set=DataBuilder(DATA_PATH)
134
135    pred_rpknn = [0] * len(data_set.y)
136    kf = KFold(n_splits=10, shuffle = True, random_state=7)
137
138    for train_index, test_index in kf.split(data_set.x):
139        #Create KNN Classifier
140        knn = KNeighborsClassifier(n_neighbors=5)
141        #print("TRAIN:", train_index, "TEST:", test_index)
142        x_train, x_test = data_set.x[train_index], data_set.x[test_index]
143        y_train, y_test = data_set.y[train_index], data_set.y[test_index]
144        #Train the model using the training sets
145        y1_train = y_train.view(-1)
146        knn.fit(x_train, y1_train)
147        #Predict the response for test dataset
148        y_pred = knn.predict(x_test)
149        #print(y_pred)
150        # Model Accuracy, how often is the classifier correct?
151        print("Accuracy in loop:", metrics.accuracy_score(y_test, y_pred))
152        c = 0
153        for idx in test_index:
154            pred_rpknn[idx] = y_pred[c]
155            c +=1
156    print("Accuracy:",metrics.accuracy_score(data_set.y, pred_rpknn))
157    print(pred_rpknn, data_set.y.reshape(1,-1))
158
159
160Accuracy in loop: 1.0
161Accuracy in loop: 0.8571428571428571
162Accuracy in loop: 0.8571428571428571
163Accuracy in loop: 1.0
164Accuracy in loop: 0.9
165Accuracy in loop: 0.9
166Accuracy in loop: 0.95
167Accuracy in loop: 1.0
168Accuracy in loop: 0.9
169Accuracy in loop: 1.0
170Accuracy: 0.9359605911330049
171

1    for %%f in (1 2 3) do (
2            for %%a in (4 5 6) do (
3                for %%b in (7 8 9) do (
4                    call :mission %%f %%a %%b
5                )
6            )
7         )
8    some gawk scripts
9    some python scripts
10    some r scripts
11    go to :exit
12
13:mission
14   sed -e 's/text1/%1/' -e 's/text2/%2/' -e 's/text3/%3/'
15   simulation.exe
16   go to :exit
17
18:exit
19start /affinity 1 C:\Users\614890\R-4.1.1\bin\Rscript.exe test1.R
20start /affinity 2 C:\Users\614890\R-4.1.1\bin\Rscript.exe test2.R
21start /affinity 3 C:\Users\614890\R-4.1.1\bin\Rscript.exe test3.R
22start /affinity 4 C:\Users\614890\R-4.1.1\bin\Rscript.exe test4.R
23
24C:\Users\614890\R-4.1.1\bin\Rscript.exe plotting.R
25setlocal
26set "lock=%temp%\wait%random%.lock"
27
28:: Launch processes asynchronously, with stream 9 redirected to a lock file.
29:: The lock file will remain locked until the script ends
30start /affinity 1 9>"%lock%1" Rscript test1.R
31start /affinity 2 9>"%lock%2" Rscript test2.R
32start /affinity 4 9>"%lock%3" Rscript test3.R
33start /affinity 8 9>"%lock%4" Rscript test4.R
34
35:Wait for all processes to finish
361>nul 2>nul ping /n 2 ::1
37for %%F in ("%lock%*") do (
38 (call ) 9>"%%F" || goto :Wait
39) 2>nul
40
41del "%lock%*"
42
43Rscript plotting.R
44

1a = [5,4]  
2b = [2,1,3]
3diff_matrix = zeros(Int8, size(b)[1], size(a)[1])
4for ia in eachindex(a)
5    for ib in eachindex(b)
6        diff_matrix[ib,ia]= a[ia] - b[ib]
7    end
8end
9println(diff_matrix)
10julia> a = [5,4]                
112-element Vector{Int64}:        
12 5                              
13 4                              
14                                
15julia> b = [2,1,3]              
163-element Vector{Int64}:        
17 2                              
18 1                              
19 3                              
20                                
21julia> permutedims(a) .- b      
223×2 Matrix{Int64}:              
23 3  2                           
24 4  3                           
25 2  1                           
26

1>>> @udf('array<string>')
2... def filter_udf(a):
3...     from builtins import filter
4...     return list(filter(lambda s: s.isascii(), a))
5... 
6
7>>> df = spark.createDataFrame([(['pyspark','பரமசிவம்'],)])
8>>> df.select(filter_udf('_1')).show()
9+--------------+
10|filter_udf(_1)|
11+--------------+
12|     [pyspark]|
13+--------------+
14