FileDownloader | Multitask、MultiThread、Breakpointresume、Highconcurrency、Simple to use、Single/NotSingleprocess | Crawler library

import React, {Component} from 'react';
import aa from "../media/AA.pdf";

export default class FileDownloader extends Component {
    render() {
        return (
            <div className="pdf">
                <a href={aa} download="aa.pdf">Download</a>
            </div>
        );
    }
}

af-ZA
am-ET
ar-SA
az-Latn-AZ
be-BY
bg-BG
bn-BD
bs-Latn-BA
ca-ES
cs-CZ
da-DK
de-DE
dotnetCampus.FileDownloader.dll
el-GR
en-GB
es-ES
es-MX
et-EE
eu-ES
fa-IR
fi-FI
file
fr-CA
fr-FR
gl-ES
ha-Latn-NG
he-IL
hi-IN
hr-HR
hu-HU
ICSharpCode.SharpZipLib.dll
ICSharpCode.SharpZipLib.pdb
id-ID
is-IS
it-IT
ja-JP
ka-GE
kk-KZ
km-KH
kn-IN
ko-KR
lo-LA
lt-LT
lv-LV
Microsoft.Web.WebView2.Core.dll
Microsoft.Web.WebView2.WinForms.dll
Microsoft.Web.WebView2.Wpf.dll
mk-MK
ml-IN
ModernWpf.Controls.dll
ModernWpf.dll
ms-MY
nb-NO
nl-NL
nn-NO
myapp.exe
myapp.exe.config
myapp.pdb
myapp.xml
pl-PL
pt-BR
pt-PT
ro-RO
ru-RU
runtimes
sk-SK
sl-SI
sq-AL
sr-Latn-RS
sv-SE
sw-KE
System.Drawing.Common.dll
System.ValueTuple.dll
ta-IN
te-IN
th-TH
tr-TR
uk-UA
uz-Latn-UZ
vi-VN
WPFUI.dll
zh-CN
zh-TW

from logging import StreamHandler, exception
from sqlalchemy import create_engine
from re import search
from pandas._config.config import options
from pandas.core.frame import DataFrame
from pandas.core.tools import numeric
import streamlit as st       
import plotly.express as pt
import pandas as pd
import numpy as np
import os
import base64

#function
def filedownloader(database):
    csv = database.to_csv(index=False)
    b64 = base64.b64encode(csv.encode()).decode() 
    href = f'<a href="data:file/csv;base64,{b64}">Download csv file</a>'
    st.markdown(href,unsafe_allow_html=True)
    return href




#basic titles

st.title("Institution's Section")
st.sidebar.subheader("Settings")

# file uploader

fl=st.sidebar.file_uploader(label="Uplaod File in csv or xlsx format", type=['csv','xlsx'])
global df
if fl is not None:
    st.title("Performance Graph")
    try:
        df=pd.read_csv(fl)
    except Exception as e:
        print(e)
        df=pd.read_excel(fl)

#write table in web page

global columns
try:
    st.write(df)
    columns = list(df.columns)
except Exception as e:
    print(e)
    st.write("Please upload a file")

#Chart select

st.sidebar.subheader("Analysis section")
ch=st.sidebar.selectbox(
    label="Select the chart type",
    options=['Scatterplots','Lineplots','Histogram','Boxplot'],
    key="chart"
)

#Plot Settings

if ch =='Scatterplots':
    st.sidebar.subheader("Scatterplot Settings")
    
    try:
        x_values=st.sidebar.selectbox('X axis',options=columns)
        y_values=st.sidebar.selectbox('Y axis',options=columns)
        plot=pt.scatter(data_frame=df,x=x_values,y=y_values)
        st.plotly_chart(plot)
    except Exception as e:
        print(e)
elif ch=='Lineplots':
    st.sidebar.subheader("Lineplot Settings")
    
    try:
        x_values=st.sidebar.selectbox('X axis',options=columns)
        y_values=st.sidebar.selectbox('Y axis',options=columns)
        plot=pt.line(data_frame=df,x=x_values,y=y_values)
        st.plotly_chart(plot)
    except Exception as e:
        print(e)
elif ch=='Histogram':
    st.sidebar.subheader("Histogram Settings")
    
    try:
        x_values=st.sidebar.selectbox('X axis',options=columns)
        y_values=st.sidebar.selectbox('Y axis',options=columns)
        plot=pt.histogram(data_frame=df,x=x_values,y=y_values)
        st.plotly_chart(plot)
    except Exception as e:
        print(e)
elif ch=='Boxplot':
    st.sidebar.subheader("Boxplot Settings")
    
    try:
        x_values=st.sidebar.selectbox('X axis',options=columns)
        y_values=st.sidebar.selectbox('Y axis',options=columns)
        plot=pt.box(data_frame=df,x=x_values,y=y_values)
        st.plotly_chart(plot)
    except Exception as e:
        print(e)

#rank select


st.sidebar.subheader("Quick Search Rank")
radio=st.sidebar.radio(
    "What you want to see?",("Top five","Bottom five")
)

#rank settings

numeric_column = df.select_dtypes(include=np.number).columns.tolist()
if(radio=='Top five'):
    st.sidebar.subheader("Rank Settings")
    try:
        val=st.sidebar.selectbox('Select rank category',options=numeric_column)
        st.title("Quick Search Result")
        st.write(df.nlargest(5,val))
    except Exception as e:
        print(e)
elif(radio=='Bottom five'):
    st.sidebar.subheader("Rank Settings")
    try:
        val=st.sidebar.selectbox('Select rank category',options=numeric_column)
        st.title("Quick Search Result")
        st.write(df.nsmallest(5,val))
    except Exception as e:
        print(e)

#search

st.sidebar.subheader("Search")
val=st.sidebar.selectbox('Search desired column',options=columns)
st.sidebar.subheader("Search keyword")
user_input = st.sidebar.text_area("Type Keyword here", 0)
try: 
    dl = df[df[val] == type(df[val][1])(user_input)]
except Exception as e:
    pass

st.sidebar.subheader("Custom plot settings")
choose=st.sidebar.selectbox(
    label="Choose plot for selected data",
    options=['Scatterplot','Linearplot','Histogram','Boxplot',]
)
if(choose == 'Scatterplot'):
    try:
        st.sidebar.subheader("Scatterplot Settings")
        x_val=st.sidebar.selectbox('X axis',options=columns,key="scatter_x")
        y_val=st.sidebar.selectbox('Y axis',options=columns,key="scatter_y")
        plots=pt.scatter(data_frame=dl,x=x_val,y=y_val)
    except Exception as e:
        pass
elif (choose == 'Linearplot'):
    try:
        st.sidebar.subheader("Scatterplot Settings")
        x_val=st.sidebar.selectbox('X axis',options=columns,key="scatter_x")
        y_val=st.sidebar.selectbox('Y axis',options=columns,key="scatter_y")
        plots=pt.line(data_frame=dl,x=x_val,y=y_val)
    except Exception as e:
        pass
elif (choose == 'Histogram'):
    try:
        st.sidebar.subheader("Scatterplot Settings")
        x_val=st.sidebar.selectbox('X axis',options=columns,key="scatter_x")
        y_val=st.sidebar.selectbox('Y axis',options=columns,key="scatter_y")
        plots=pt.histogram(data_frame=dl,x=x_val,y=y_val)
    except Exception as e:
        pass
elif (choose == 'Boxplot'):
    try:
        st.sidebar.subheader("Scatterplot Settings")
        x_val=st.sidebar.selectbox('X axis',options=columns,key="scatter_x")
        y_val=st.sidebar.selectbox('Y axis',options=columns,key="scatter_y")
        plots=pt.box(data_frame=dl,x=x_val,y=y_val)
    except Exception as e:
        pass

#chart and search button

bt=st.sidebar.button("Search and Apply")
if(bt):
    st.title("Search Results")
    st.write(dl)
    st.title("Searh Analysis")
    st.plotly_chart(plots)

#Modify database
st.sidebar.subheader("Select Column to drop null values")
val1=st.sidebar.multiselect(
    label="Selet Columns",
    options=columns
)
bt_null_r=st.sidebar.button("Drop Null values")
if(bt_null_r):
    mdf=df.dropna(subset=val1, how='all')
    filedownloader(mdf)

# drop selected column
st.sidebar.subheader("Select Column to drop null values")
val2=st.sidebar.multiselect(
    label="Selet Columns",
    options=columns,key="column_multi"
)
bt_c=st.sidebar.button("Drop Columns")
if(bt_c):
    mdf=df.drop(val2,axis=1)
    filedownloader(mdf)
**#creating sql engine
engine=create_engine('mysql://root:Soumik18@@localhost:3306/college')
df.to_sql('Students',con=engine)**

class FileDownloader {

    companion object {
        private const val BUFFER_LENGTH_BYTES = 1024 * 8
        private const val HTTP_TIMEOUT = 30
    }

    private var okHttpClient: OkHttpClient

    init {
        val okHttpBuilder = okHttpClient.newBuilder()
            .connectTimeout(HTTP_TIMEOUT.toLong(), TimeUnit.SECONDS)
            .readTimeout(HTTP_TIMEOUT.toLong(), TimeUnit.SECONDS)
        this.okHttpClient = okHttpBuilder.build()
    }

    fun download(url: String, file: File): {
            val request = Request.Builder().url(url).build()
            val response = okHttpClient.newCall(request).execute()
            val body = response.body
            val responseCode = response.code
            if (responseCode >= HttpURLConnection.HTTP_OK &&
                responseCode < HttpURLConnection.HTTP_MULT_CHOICE &&
                body != null) {
                val length = body.contentLength()
                body.byteStream().apply {
                    file.outputStream().use { fileOut ->
                        var bytesCopied = 0
                        val buffer = ByteArray(BUFFER_LENGTH_BYTES)
                        var bytes = read(buffer)
                        while (bytes >= 0) {
                            fileOut.write(buffer, 0, bytes)
                            bytesCopied += bytes
                            bytes = read(buffer)
                        }
                    }
                }
            } else {
                throw IllegalArgumentException("Error occurred when do http get $url")
            }
        }
    }
}

public function __construct(FileDownloader $fileDownloader) {
    $this->fileDownloader = $fileDownloader;    
}

public function getProfilePicturePresignedUrl(): string
{
    return $this->fileDownloader->generatePresignedUrl($this->profilePicturePath);
}

public function getUserInfo(FileDownloader $fileDownloader)
{
    $user = $this->getUser();
    
    $user->setProfilePicturePresignedUrl($fileDownloader->generatePresignedUrl($user->getProfilePicturePath()));

    return $this->createResponse($user);
}

public async static void DownloadAllFiles(Data clientData)
{
    data = clientData;

    data.OutputFolderExists();


    // Deserialize .json file and get ClipInfo list
    List<ClipInfo> clips = JsonConvert.DeserializeObject<List<ClipInfo>>(File.ReadAllText(data.JsonFile));
            
    tasks = new List<Task>();

    foreach(ClipInfo clip in clips)
    {
        tasks.Add(DownloadFilesAsync(clip));
    }

    await Task.WhenAll(tasks);
}

private async static Task DownloadFilesAsync(ClipInfo clip)
{
    WebClient client = new WebClient();
    string url = GetClipURL(clip);
    string filepath = data.OutputPath + clip.id + ".mp4";

    await client.DownloadFileTaskAsync(new Uri(url), filepath);
}

using System;
using System.Collections.Generic;
using System.Text;

namespace Downloader
{
    public class ClipInfo
    {
        public string id { get; set; }
        public string url { get; set; }
        public string embed_url { get; set; }
        public string broadcaster_id { get; set; }
        public string broadcaster_name { get; set; }
        public string creator_id { get; set; }
        public string creator_name { get; set; }
        public string video_id { get; set; }
        public string game_id { get; set; }
        public string language { get; set; }
        public string title { get; set; }
        public int view_count { get; set; }
        public DateTime created_at { get; set; }
        public string thumbnail_url { get; set; }
    }
}

namespace Downloader
{
    public class Pagination
    {
        public string cursor { get; set; }
    }

}

using System.Collections.Generic;

namespace Downloader
{
    public class Root
    {
        public List<ClipInfo> data { get; set; }
        public Pagination pagination { get; set; }
    }
}

using System;
using System.IO;

namespace Downloader
{
    public class Data
    {
        private static string directory = Directory.GetCurrentDirectory();
        private readonly static string defaultJsonFile = directory + @"\clips.json";
        private readonly static string defaultOutputPath = directory + @"\Clips\";
        private readonly static string clipsLink = "https://api.twitch.tv/helix/clips?";

        public string OutputPath { get; set; }
        public string JsonFile { get; set; }
        public string ClientID { get; private set; }
        public string Authentication { get; private set; }
        public string QueryURL { get; private set; }
    

        public Data()
        {
            OutputPath = defaultOutputPath;
            JsonFile = defaultJsonFile;
        }
        public Data(string clientID, string authentication)
        {
            ClientID = clientID;
            Authentication = authentication;
            OutputPath = defaultOutputPath;
            JsonFile = defaultJsonFile;
        }
        public Data(string clientID, string authentication, string outputPath)
        {
            ClientID = clientID;
            Authentication = authentication;
            OutputPath = directory + @"\" + outputPath + @"\";
            JsonFile = OutputPath + outputPath + ".json";
        }

        public void GetQuery()
        {
            Console.Write("Please enter your query: ");
            QueryURL = clipsLink + Console.ReadLine();
        }

        public void GetClientID()
        {
            Console.WriteLine("Enter your client ID");
            ClientID = Console.ReadLine();
        }

        public void GetAuthentication()
        {
            Console.WriteLine("Enter your Authentication");
            Authentication = Console.ReadLine();
        }

        public void OutputFolderExists()
        {
            if (!Directory.Exists(OutputPath))
            {
                Directory.CreateDirectory(OutputPath);
            }
        }

    }
}

using System;
using System.IO;
using System.Net.Http.Headers;
using System.Net.Http;
using System.Threading.Tasks;
using Newtonsoft.Json;
using System.Linq;


namespace Downloader
{
    public static class JsonGenerator
    {
        // This class has no constructor.
        // You call the Generate methods, passing in all required data.
        // The file will then be generated.
        private static Data data;

        public static async Task Generate(Data clientData)
        {
            data = clientData;
            string responseContent = null;

            // Loop that runs until the api request goes through
            bool authError = true;
            while (authError)
            {
                authError = false;
                try
                {
                    responseContent = await GetHttpResponse();
                }
                catch (HttpRequestException)
                {
                    Console.WriteLine("Invalid authentication, please enter client-ID and authentication again!");
                    data.GetClientID();
                    data.GetAuthentication();

                    authError = true;
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);
                    authError = true;
                }
            }

            data.OutputFolderExists();
            GenerateJson(responseContent);
        }

        // Returns the contents of the resopnse to the api call as a string
        private static async Task<string> GetHttpResponse()
        {
            // Creating client
            HttpClient client = new HttpClient();

            if (data.QueryURL == null)
            {
                data.GetQuery();
            }


            // Setting up request
            HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, data.QueryURL);

            // Adding Headers to request
            requestMessage.Headers.Add("client-id", data.ClientID);
            requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", data.Authentication);

            // Receiving response to the request
            HttpResponseMessage responseMessage = await client.SendAsync(requestMessage);

            // Gets the content of the response as a string
            string responseContent = await responseMessage.Content.ReadAsStringAsync();

            return responseContent;
        }

        // Generates or adds to the .json file that contains data on each clip
        private static void GenerateJson(string responseContent)
        {
            // Parses the data from the response to the api request
            Root responseResult = JsonConvert.DeserializeObject<Root>(responseContent);

            // If the file doesn't exist, we need to create it and add a '[' at the start
            if (!File.Exists(data.JsonFile))
            {
                FileStream file = File.Create(data.JsonFile);
                file.Close();
                // The array of json objects needs to be wrapped inside []
                File.AppendAllText(data.JsonFile, "[\n");
            }
            else
            {
                // For a pre-existing .json file, The last object won't have a comma at the
                // end of it so we need to add it now, before we add more objects
                string[] jsonLines = File.ReadAllLines(data.JsonFile);
                File.WriteAllLines(data.JsonFile, jsonLines.Take(jsonLines.Length - 1).ToArray());
                File.AppendAllText(data.JsonFile, ",");
            }

            // If the file already exists, but there was no [ at the start for whatever reason,
            // we need to add it
            if (File.ReadAllText(data.JsonFile).Length == 0 || File.ReadAllText(data.JsonFile)[0] != '[')
            {
                File.WriteAllText(data.JsonFile, "[\n" + File.ReadAllText(data.JsonFile));
            }

            string json;

            // Loops through each ClipInfo object that the api returned
            for (int i = 0; i < responseResult.data.Count; i++)
            {
                // Serializes the ClipInfo object into a json style string
                json = JsonConvert.SerializeObject(responseResult.data[i]);

                // Adds the serialized contents of ClipInfo to the .json file
                File.AppendAllText(data.JsonFile, json);

                if (i != responseResult.data.Count - 1)
                {
                    // All objects except the last require a comma at the end of the
                    // object in order to correctly format the array of json objects
                    File.AppendAllText(data.JsonFile, ",");
                }

                // Adds new line after object entry
                File.AppendAllText(data.JsonFile, "\n");
            }
            // Adds the ] at the end of the file to close off the json objects array
            File.AppendAllText(data.JsonFile, "]");
        }
    }
}

using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Threading.Tasks;

namespace Downloader
{
    public class FileDownloader
    {
        private static Data data;
        private static List<Task> tasks;
        public async static void DownloadAllFiles(Data clientData)
        {
            data = clientData;

            data.OutputFolderExists();


            // Deserialize .json file and get ClipInfo list
            List<ClipInfo> clips = JsonConvert.DeserializeObject<List<ClipInfo>>(File.ReadAllText(data.JsonFile));

            tasks = new List<Task>();

            foreach (ClipInfo clip in clips)
            {
                tasks.Add(DownloadFilesAsync(clip));
            }

            await Task.WhenAll(tasks);
        }

        private static void GetData()
        {
            if (data.ClientID == null)
            {
                data.GetClientID();
            }
            if (data.Authentication == null)
            {
                data.GetAuthentication();
            }
            if (data.QueryURL == null)
            {
                data.GetQuery();
            }
        }

        private static string GetClipURL(ClipInfo clip)
        {
            // Example thumbnail URL:
            // https://clips-media-assets2.twitch.tv/AT-cm%7C902106752-preview-480x272.jpg
            // You can get the URL of the location of clip.mp4
            // by removing the -preview.... from the thumbnail url */

            string url = clip.thumbnail_url;
            url = url.Substring(0, url.IndexOf("-preview")) + ".mp4";
            return url;
        }
            
        private async static Task DownloadFilesAsync(ClipInfo clip)
        {
            WebClient client = new WebClient();
            string url = GetClipURL(clip);
            string filepath = data.OutputPath + clip.id + ".mp4";

            await client.DownloadFileTaskAsync(new Uri(url), filepath);
        }

        private static void FileDownloadComplete(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
        {
            tasks.Remove((Task)sender);
        }
    }
}

using System;
using System.Threading.Tasks;
using Downloader;

namespace ClipDownloader
{
    class Program
    {
        private static string clientID = "{your_client_id}";
        private static string authentication = "{your_authentication}";
        async static Task Main(string[] args)
        {
            Console.WriteLine("Enter your output path");
            string outputPath = Console.ReadLine();


            Data data = new Data(clientID, authentication, outputPath);
            Console.WriteLine(data.OutputPath);

            //await JsonGenerator.Generate(data);
            FileDownloader.DownloadAllFiles(data);
        }
    }
}

@Autowired
 private FileDownloader fileDownloader;

 @Autowired
 private StepBuilderFactory stepBuilderFactory;

 @Bean
 public Step downloadFirstFileStep() {
   return stepBuilderFactory.get("downloadFirstFileStep")
       .tasklet(firstFileDownloadTasklet(null)).build();
 }

 @Bean
 @StepScope
 public FileDownloadTasklet firstFileDownloadTasklet(
     @Value("#{jobParameters['firstFile']}") String fileName) {
   return new FileDownloadTasklet(fileDownloader, fileName);
 }

 @Bean
 public Step downloadSecondFileStep() {
   return stepBuilderFactory.get("downloadSecondFileStep")
       .tasklet(secondFileDownloadTasklet(null)).build();
 }

 @Bean
 @StepScope
 public FileDownloadTasklet secondFileDownloadTasklet(
     @Value("#{jobParameters['secondFile']}") String fileName) {
   return new FileDownloadTasklet(fileDownloader, fileName);
 }

 let url = URL(string: v)
 FileDownloader.loadFileAsync(url: url!) { (path, error) in
    print("PDF File downloaded to : \(path!)")

    guard let unzipPath = self.tempUnzipPath() else {
       return
    }
    print("Unzip path:", unzipPath)

    let success: Bool = SSZipArchive.unzipFile(atPath: path!,
                                               toDestination: unzipPath,
                                               preserveAttributes: true,
                                               overwrite: true,
                                               nestedZipLevel: 1,
                                               password: nil,
                                               error: nil,
                                               delegate: nil,
                                               progressHandler: nil,
                                               completionHandler: nil)
    if success != false {
       print("Success unzip")
    } else {
       print("No success unzip")
       return
    }

    var items: [String]
    do {
       items = try FileManager.default.contentsOfDirectory(atPath: unzipPath)
       print("array item for unzip", items[0])               
    } catch {
       return
    }

    for v in items[0] {
       print("vv index.html", v)
       self.buildAry.append(v)
    }

    print("build ary", self.buildAry)
    let ss = self.buildAry[0]
    print("index html", ss)
 }                              

 Success unzip
 array item for unzip build
vv index.html b
vv index.html u
vv index.html i
vv index.html l
vv index.html d
 build ary 2020-02-07 19:40:44.480754+0530[9847:2135988] [Process] kill() returned unexpected error 1
["b", "u", "i", "l", "d"]
 index html b