responses | A utility for mocking out the Python Requests library | Mock library
kandi X-RAY | responses Summary
Support
Quality
Security
License
Reuse
- Find the first response matching the given request
- Return true if url matches other
- Return True if the request matches this request
- Returns True if match matches match
- Get a response
- Get the headers as a dictionary
- Handle HTTP response body
- Construct an HTTPResponse object
- Implements an upsert
- Replace an existing request
- Add a new response
- Creates a function that accepts json params
- Create a string representation of a nested dictionary
- Return a filtered version of dict1
- Creates a decorator to activate a function
- Wrap a function or response
- Create a function to match urlencoded parameters
- Decorator to record a function
- Assert that url is called once
- Add documents from a file
- Handle a request
- Returns an HTTPResponse object
- Returns a function that will match the kwargs
- Create a function to match query parameters
- Creates a function that matches the request headers
- Creates a function that matches the query string
responses Key Features
responses Examples and Code Snippets
{!../../../docs_src/additional_responses/tutorial001.py!}
**FastAPI** will take the Pydantic model from there, generate the `JSON Schema`, and put it in the correct place. The correct place is: * In the key `content`, that has as value another JSON object (`dict`) that contains: * A key with the media type, e.g. `application/json`, that contains as value another JSON object, that contains: * A key `schema`, that has as the value the JSON Schema from the model, here's the correct place. * **FastAPI** adds a reference here to the global JSON Schemas in another place in your OpenAPI instead of including it directly. This way, other applications and clients can use those JSON Schemas directly, provide better code generation tools, etc.
{ "responses": { "404": { "description": "Additional Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Message" } } } }, "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Item" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }
{ "components": { "schemas": { "Message": { "title": "Message", "required": [ "message" ], "type": "object", "properties": { "message": { "title": "Message", "type": "string" } } }, "Item": { "title": "Item", "required": [ "id", "value" ], "type": "object", "properties": { "id": { "title": "Id", "type": "string" }, "value": { "title": "Value", "type": "string" } } }, "ValidationError": { "title": "ValidationError", "required": [ "loc", "msg", "type" ], "type": "object", "properties": { "loc": { "title": "Location", "type": "array", "items": { "type": "string" } }, "msg": { "title": "Message", "type": "string" }, "type": { "title": "Error Type", "type": "string" } } }, "HTTPValidationError": { "title": "HTTPValidationError", "type": "object", "properties": { "detail": { "title": "Detail", "type": "array", "items": { "$ref": "#/components/schemas/ValidationError" } } } } } } }
http http://127.0.0.1:8000/snippets/ HTTP/1.1 200 OK ... [ { "id": 1, "title": "", "code": "foo = \"bar\"\n", "linenos": false, "language": "python", "style": "friendly" }, { "id": 2, "title": "", "code": "print(\"hello, world\")\n", "linenos": false, "language": "python", "style": "friendly" } ]
http http://127.0.0.1:8000/snippets/ Accept:application/json # Request JSON http http://127.0.0.1:8000/snippets/ Accept:text/html # Request HTML
http http://127.0.0.1:8000/snippets.json # JSON suffix http http://127.0.0.1:8000/snippets.api # Browsable API suffix
# POST using form data http --form POST http://127.0.0.1:8000/snippets/ code="print(123)" { "id": 3, "title": "", "code": "print(123)", "linenos": false, "language": "python", "style": "friendly" } # POST using JSON http --json POST http://127.0.0.1:8000/snippets/ code="print(456)" { "id": 4, "title": "", "code": "print(456)", "linenos": false, "language": "python", "style": "friendly" }
from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response from snippets.models import Snippet from snippets.serializers import SnippetSerializer @api_view(['GET', 'POST']) def snippet_list(request): """ List all code snippets, or create a new snippet. """ if request.method == 'GET': snippets = Snippet.objects.all() serializer = SnippetSerializer(snippets, many=True) return Response(serializer.data) elif request.method == 'POST': serializer = SnippetSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
@api_view(['GET', 'PUT', 'DELETE']) def snippet_detail(request, pk): """ Retrieve, update or delete a code snippet. """ try: snippet = Snippet.objects.get(pk=pk) except Snippet.DoesNotExist: return Response(status=status.HTTP_404_NOT_FOUND) if request.method == 'GET': serializer = SnippetSerializer(snippet) return Response(serializer.data) elif request.method == 'PUT': serializer = SnippetSerializer(snippet, data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) elif request.method == 'DELETE': snippet.delete() return Response(status=status.HTTP_204_NO_CONTENT)
function onDigits(req, res) { res.writeHead(200, { 'Content-Type': 'text/event-stream; charset=utf-8', 'Cache-Control': 'no-cache' }); let i = 0; let timer = setInterval(write, 1000); write(); function write() { i++; if (i == 4) { res.write('event: bye\ndata: пока-пока\n\n'); clearInterval(timer); res.end(); return; } res.write('data: ' + i + '\n\n'); } }
@Bean public RouterFunction responseRoute(@Autowired ServerHandler handler) { return RouterFunctions.route(RequestPredicates.GET("/functional-reactive/periodic-foo"), handler::useHandler) .andRoute(RequestPredicates.GET("/functional-reactive/periodic-foo-2"), handler::useHandlerFinite); }
playerdata["listings"].append(newPlayerData)
playerdata["listings"].extend(newPlayerData)
from sklearn.neighbors import radius_neighbors_graph
# Your example data in runnable format
dx = np.array([2.63370612e-01, 3.48350511e-01, -1.23379511e-02,
6.63767411e+00, 1.32910697e+01, 8.75469902e+00])
dy = np.array([0.33889825, 0.21808108, 0.50170807,
8.95542985, -9.84251952, -16.38661054])
dz = np.array([-0.26469788, -0.10382767, -0.16625317,
-4.84708218, -13.77888398, 10.42730599])
# Build a coordinate matrix with columns x, y, z, with one star per row
X = np.column_stack([dx, dy, dz])
print(X)
[[ 2.63370612e-01 3.38898250e-01 -2.64697880e-01]
[ 3.48350511e-01 2.18081080e-01 -1.03827670e-01]
[-1.23379511e-02 5.01708070e-01 -1.66253170e-01]
[ 6.63767411e+00 8.95542985e+00 -4.84708218e+00]
[ 1.32910697e+01 -9.84251952e+00 -1.37788840e+01]
[ 8.75469902e+00 -1.63866105e+01 1.04273060e+01]]
# Find the neighbours of each star, restricted to distance lower than radius=1.2
C = radius_neighbors_graph(X, 1.2)
# C is the connectivity matrix in Compressed Sparse Row (CSR) format.
# For demonstration purposes, convert CSR matrix to dense representation
# as a numpy matrix
C.todense()
matrix([[0., 1., 1., 0., 0., 0.],
[1., 0., 1., 0., 0., 0.],
[1., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0.]])
for df in motifs:
df['first_elements'] = df.iloc[:, 9].apply(lambda li: [x[0] for x in li])
[ TF MotifID Enrichment \
AUC
0 Arid3a tfdimers__MD00454 0.064714
1 Arnt taipale_cyt_meth__SREBF1_NTCACGTGAN_eDBD 0.060952
2 Arnt cisbp__M4597 0.070117
3 Arnt hocomoco__ATF3_HUMAN.H11MO.0.A 0.067057
4 Arnt cisbp__M4552 0.062478
\
Annotation
0 motif is annotated for orthologous gene ENSG00...
1 motif similar to transfac_public__M00539 ('V$A...
2 gene is annotated for similar motif transfac_p...
3 gene is annotated for similar motif transfac_p...
4 gene is annotated for similar motif transfac_p...
\
Context MotifSimilarityQvalue
0 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000000
1 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000031
2 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000799
3 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000575
4 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000358
\
NES OrthologousIdentity RankAtMax
0 3.326403 0.809444 1185
1 3.120903 1.000000 298
2 3.922071 1.000000 901
3 3.654649 1.000000 865
4 3.254340 1.000000 4637
\
TargetGenes
0 [(Hmgb1, 0.745314226221018), (Zfp771, 0.676482...
1 [(Clcn6, 0.5838135470801639), (Ptprs, 2.580731...
2 [(Ptprs, 0.5838135470801639), (Clcn6, 2.580731...
3 [(Ptprs, 0.5838135470801639), (Clcn6, 2.580731...
4 [(Clcn6, 0.5838135470801639), (Ptprs, 2.580731...
first_elements
0 [Hmgb1, Zfp771, Irgc1, Bcl11a, Sh3kbp1, Traf3,...
1 [Clcn6, Ptprs, Erp29, Lin52, Smndc1, Scarb1, R...
2 [Ptprs, Clcn6, Pde7a, Smndc1, Ppp2r2a, Gzf1, P...
3 [Ptprs, Clcn6, Pde7a, Smndc1, Gzf1, Atg10, Erp...
4 [Clcn6, Ptprs, Lin52, Erp29, Smndc1, Rnf146, M... , TF MotifID Enrichment \
AUC
0 Atf3 dbcorrdb__JUN__ENCSR000EGH_1__m1 0.068472
1 Atf3 dbcorrdb__JUND__ENCSR000EGN_1__m1 0.072980
2 Atf3 cisbp__M5050 0.059033
3 Atf3 dbcorrdb__eGFP-JUNB__ENCSR000DJY_1__m1 0.074232
4 Atf3 dbcorrdb__FOSL1__ENCSR000BMV_1__m1 0.076303
\
Annotation
0 gene is annotated for similar motif hocomoco__...
1 gene is orthologous to ENSG00000162772 in H. s...
2 gene is annotated for similar motif hocomoco__...
3 gene is annotated for similar motif hocomoco__...
4 gene is annotated for similar motif hocomoco__...
\
Context MotifSimilarityQvalue
0 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000773
1 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000045
2 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000608
3 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000006
4 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000004
\
NES OrthologousIdentity RankAtMax
0 4.024299 1.000000 481
1 4.467018 0.950276 1112
2 3.097418 1.000000 829
3 4.589883 1.000000 634
4 4.793295 1.000000 762
\
TargetGenes
0 [(Tagln2, 1.6868254779790988), (Junb, 2.131165...
1 [(Kdm6b, 1.6868254779790988), (Junb, 2.1311655...
2 [(Junb, 1.6868254779790988), (Oser1, 2.1311655...
3 [(Kdm6b, 1.6868254779790988), (Junb, 2.1311655...
4 [(Junb, 1.6868254779790988), (Ptp4a2, 2.131165...
first_elements
0 [Tagln2, Junb, Pim1, Mir155hg, Kdm6b, Vcpip1, ...
1 [Kdm6b, Junb, Tagln2, Dusp1, Mir155hg, Sec11c,...
2 [Junb, Oser1, Tagln2, Lgals3, Bach1, Csrnp1, K...
3 [Kdm6b, Junb, Ptp4a2, Sec11c, Lgals3, Pim1, Ta...
4 [Junb, Ptp4a2, Pim1, Kdm6b, Sec11c, Vcpip1, Di... , TF MotifID Enrichment \
AUC
0 Arid3a cisbp__M1879 0.066822
1 Arid3a swissregulon__hs__FOXA2.p3 0.066466
2 Arid3a homer__AAAGTAAACA_FOXA1_GSE26831 0.067375
3 Arnt cisbp__M5633 0.069684
4 Arnt cisbp__M5866 0.068370
\
Annotation
0 gene is orthologous to ENSG00000116017 in H. s...
1 gene is orthologous to ENSG00000116017 in H. s...
2 motif similar to dbcorrdb__ARID3A__ENCSR000EDP...
3 gene is annotated for similar motif transfac_p...
4 gene is annotated for similar motif transfac_p...
\
Context MotifSimilarityQvalue
0 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000230
1 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000230
2 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000008
3 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000417
4 (activating, mm10__refseq-r80__10kb_up_and_dow... 0.000133
\
NES OrthologousIdentity RankAtMax
0 3.048012 0.798669 414
1 3.020346 0.798669 352
2 3.090921 0.809444 398
3 3.726068 1.000000 945
4 3.612595 1.000000 499
\
TargetGenes
0 [(Arid3a, 1.0814455429211889), (Pogz, 0.624427...
1 [(Arid3a, 1.0814455429211889), (Pogz, 0.624427...
2 [(Pogz, 1.0814455429211889), (Itpr1, 0.6244276...
3 [(Clcn6, 0.8095717882553205), (Kansl1, 1.58349...
4 [(Clcn6, 0.8095717882553205), (Lin52, 1.583490...
first_elements
0 [Arid3a, Pogz, Ago4, Taf1b, Itpr1, Hmgb1, Sh3k...
1 [Arid3a, Pogz, Ago4, Taf1b, Itpr1, Hmgb1, Sh3k...
2 [Pogz, Itpr1, Arid3a, Sh3kbp1, Hmgb1, Clpx, Me...
3 [Clcn6, Kansl1, Lin52, Ptprs, Asna1, Ccdc91, E...
4 [Clcn6, Lin52, Ptprs, Kansl1, Asna1, Smndc1, R... ]
Trending Discussions on responses
Trending Discussions on responses
QUESTION
I decided today that I'm going to use Strapi as my headless CMS for my portfolio, I've bumped into some issues though, which I just seem to not be able to find a solution to online. Maybe I'm just too clueless to actually find the real issue.
I have set up a schema for my projects that will be stored in Strapi (everything done in the web), but I've had some issues with my custom components, and that is, they are not part of the API responses when I run it through Postman. (Not just empty keys but not included in the response at all). All other fields, that are not components, are filled out as expected.
At first I thought it might have to do with the permissions, but everything is enabled so it can't be that, I also tried looking into the API in the code, but that logging the answer there didn't include the components either.
Here is an image of some of the fields in the schema, but more importantly the components that are not included in the response.
So my question is, do I need to create some sort of a parser or anything in the project to be able to include these fields, or why are they not included?
ANSWER
Answered 2021-Dec-06 at 20:22I had the same problem and was able to fix it by adding populate=* to the end of the API endpoint.
For example:
http://localhost:1337/api/test?populate=*
More info here: https://forum.strapi.io/t/cannot-see-media-field-in-my-endpoint-for-my-content-type/13082/2
edit: Only first-level relations are populated with populate=* . Use the LHS bracket syntax (i.e. [populate]=*) to populate deeper:
For example:
http://localhost:1337/api/test?populate[TestExamples][populate]=*
More info here if you go to Relations population: https://docs.strapi.io/developer-docs/latest/developer-resources/database-apis-reference/rest-api.html#fields-selection
QUESTION
I want to download/scrape 50 million log records from a site. Instead of downloading 50 million in one go, I was trying to download it in parts like 10 million at a time using the following code but it's only handling 20,000 at a time (more than that throws an error) so it becomes time-consuming to download that much data. Currently, it takes 3-4 mins to download 20,000 records with the speed of 100%|██████████| 20000/20000 [03:48<00:00, 87.41it/s]
so how to speed it up?
import asyncio
import aiohttp
import time
import tqdm
import nest_asyncio
nest_asyncio.apply()
async def make_numbers(numbers, _numbers):
for i in range(numbers, _numbers):
yield i
n = 0
q = 10000000
async def fetch():
# example
url = "https://httpbin.org/anything/log?id="
async with aiohttp.ClientSession() as session:
post_tasks = []
# prepare the coroutines that poat
async for x in make_numbers(n, q):
post_tasks.append(do_get(session, url, x))
# now execute them all at once
responses = [await f for f in tqdm.tqdm(asyncio.as_completed(post_tasks), total=len(post_tasks))]
async def do_get(session, url, x):
headers = {
'Content-Type': "application/x-www-form-urlencoded",
'Access-Control-Allow-Origin': "*",
'Accept-Encoding': "gzip, deflate",
'Accept-Language': "en-US"
}
async with session.get(url + str(x), headers=headers) as response:
data = await response.text()
print(data)
s = time.perf_counter()
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(fetch())
except:
print("error")
elapsed = time.perf_counter() - s
# print(f"{__file__} executed in {elapsed:0.2f} seconds.")
Traceback (most recent call last):
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 986, in _wrap_create_connection
return await self._loop.create_connection(*args, **kwargs) # type: ignore[return-value] # noqa
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 1056, in create_connection
raise exceptions[0]
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 1041, in create_connection
sock = await self._connect_sock(
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 955, in _connect_sock
await self.sock_connect(sock, address)
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\proactor_events.py", line 702, in sock_connect
return await self._proactor.connect(sock, address)
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 328, in __wakeup
future.result()
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\windows_events.py", line 812, in _poll
value = callback(transferred, key, ov)
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\windows_events.py", line 599, in finish_connect
ov.getresult()
OSError: [WinError 121] The semaphore timeout period has expired
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\SGM\Desktop\xnet\x3stackoverflow.py", line 136, in
loop.run_until_complete(fetch())
File "C:\Users\SGM\AppData\Roaming\Python\Python39\site-packages\nest_asyncio.py", line 81, in run_until_complete
return f.result()
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 256, in __step
result = coro.send(None)
File "C:\Users\SGM\Desktop\xnet\x3stackoverflow.py", line 88, in fetch
response = await f
File "C:\Users\SGM\Desktop\xnet\x3stackoverflow.py", line 37, in _wait_for_one
return f.result()
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 258, in __step
result = coro.throw(exc)
File "C:\Users\SGM\Desktop\xnet\x3stackoverflow.py", line 125, in do_get
async with session.get(url + str(x), headers=headers) as response:
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 1138, in __aenter__
self._resp = await self._coro
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 535, in _request
conn = await self._connector.connect(
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 542, in connect
proto = await self._create_connection(req, traces, timeout)
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 907, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 1206, in _create_direct_connection
raise last_exc
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 1175, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "C:\Users\SGM\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\connector.py", line 992, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host example.com:80 ssl:default [The semaphore timeout period has expired]
ANSWER
Answered 2022-Feb-27 at 14:37If it's not the bandwidth that limits you (but I cannot check this), there is a solution less complicated than the celery and rabbitmq but it is not as scalable as the celery and rabbitmq, it will be limited by your number of CPU.
Instead of splitting calls on celery workers, you split them on multiple processes.
I modified the fetch
function like this:
async def fetch(start, end):
# example
url = "https://httpbin.org/anything/log?id="
async with aiohttp.ClientSession() as session:
post_tasks = []
# prepare the coroutines that poat
# use start and end arguments here!
async for x in make_numbers(start, end):
post_tasks.append(do_get(session, url, x))
# now execute them all at once
responses = [await f for f in
tqdm.tqdm(asyncio.as_completed(post_tasks), total=len(post_tasks))]
and I modified the main processes:
import concurrent.futures
from itertools import count
def one_executor(start, end):
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(fetch(start, end))
except:
print("error")
if __name__ == '__main__':
s = time.perf_counter()
# Change the value to the number of core you want to use.
max_worker = 4
length_by_executor = q // max_worker
with concurrent.futures.ProcessPoolExecutor(max_workers=max_worker) as executor:
for index_min in count(0, length_by_executor):
# no matter with duplicated indexes due to the use of
# range in make_number function.
index_max = min(index_min + length_by_executor, q)
executor.submit(one_executor, index_min, index_max)
if index_max == q:
break
elapsed = time.perf_counter() - s
print(f"executed in {elapsed:0.2f} seconds.")
Here the result I get (with the value of q
set to 10_000
):
1 worker: executed in 13.90 seconds.
2 workers: executed in 7.24 seconds.
3 workers: executed in 6.82 seconds.
I don't work on the tqdm
progress bar, with the current solution, two bars will be displayed (but I think tqdm works well with multi processes).
QUESTION
I am running a Spring Boot app that uses WebClient for both non-blocking and blocking HTTP requests. After the app has run for some time, all outgoing HTTP requests seem to get stuck.
WebClient is used to send requests to multiple hosts, but as an example, here is how it is initialized and used to send requests to Telegram:
WebClientConfig:
@Bean
public ReactorClientHttpConnector httpClient() {
HttpClient.create(ConnectionProvider.builder("connectionProvider").build())
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectTimeout)
.responseTimeout(Duration.ofMillis(responseTimeout));
return new ReactorClientHttpConnector(httpClient);
}
The same ReactorClientHttpConnector is used for all WebClients.
TelegramClient:
@Autowired
ReactorClientHttpConnector httpClient;
WebClient webClient;
RateLimiter rateLimiter;
@PostConstruct
public void init() {
webClient = WebClient.builder()
.clientConnector(httpClient)
.baseUrl(telegramUrl)
.build();
rateLimiter = RateLimiter.of("telegram-rate-limiter",
RateLimiterConfig.custom()
.limitRefreshPeriod(Duration.ofMinutes(1))
.limitForPeriod(20)
.build());
}
public void sendMessage(@PathVariable("token") String token, @RequestParam("chat_id") long chatId, @RequestParam("text") String message) {
webClient.post().uri(String.format("/bot%s/sendMessage", token))
.contentType(MediaType.APPLICATION_JSON)
.body(BodyInserters.fromFormData("chat_id", String.valueOf(chatId))
.with("text", message))
.retrieve()
.bodyToMono(Void.class)
.transformDeferred(RateLimiterOperator.of(rateLimiter))
.block();
}
The RateLimiter is used to ensure the number of requests does not exceed 20 a minute as specified in the Telegram API.
When the app is started, all requests are resolved normally as expected. But after some time has passed, all requests seem to get stuck. The amount of time needed for this to happen can vary from a few hours to a few days. It happens for all requests to different hosts and is easily noticeable when messages from the TelegramBot stops. Once the requests get stuck, they are stuck indefinitely and I have to restart the app to get it working again.
There are no exceptions in the log that seem to have caused this. Since I maintain a queue for my telegram messages, I can see the point in time when the requests stop when the number of messages in the queue steadily increases and when errors happen in the other processes that are waiting for the requests to resolve.
It does not seem like the requests are even sent out as the connect timeout and response timeout that I have set do not take effect.
I had previously also tried setting idle time to 0 but that did not solve the problem
@Bean
public ReactorClientHttpConnector httpClient() {
HttpClient httpClient = HttpClient.create(ConnectionProvider.builder("connectionProvider").maxConnections(1000).maxIdleTime(Duration.ofSeconds(0)).build())
HttpClient httpClient = HttpClient.newConnection()
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectTimeout)
.responseTimeout(Duration.ofMillis(responseTimeout));
return new ReactorClientHttpConnector(httpClient);
}
Update:
I enabled metrics and viewed it using micrometer when it got stuck. Interestingly, it shows that there is one connection for Telegram, but also shows no connections on idle, pending or active.
reactor_netty_connection_provider_idle_connections{id="-1268283746",name="connectionProvider",remote_address="api.telegram.org:443",} 0.0
reactor_netty_connection_provider_pending_connections{id="-1268283746",name="connectionProvider",remote_address="api.telegram.org:443",} 0.0
reactor_netty_connection_provider_active_connections{id="-1268283746",name="connectionProvider",remote_address="api.telegram.org:443",} 0.0
reactor_netty_connection_provider_total_connections{id="-1268283746",name="connectionProvider",remote_address="api.telegram.org:443",} 1.0
Could the issue be this missing connection?
Update 2:
I thought this might be related to this other issue: Closing Reactor Netty connection on error status codes
So I updated my HttpClient to this:
@Bean
public ReactorClientHttpConnector httpClient() {
HttpClient httpClient = HttpClient.create(ConnectionProvider.builder("connectionProvider").metrics(true).build())
.doAfterResponseSuccess((r, c) -> c.dispose())
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectTimeout)
.responseTimeout(Duration.ofMillis(responseTimeout));
return new ReactorClientHttpConnector(httpClient);
}
But all that seemed to do is accelerate the occurrence of the problem. Just like before, the active, pending and idle connections do not add up to the total connections. The total is always greater than the other 3 metrics added together.
Update 3: I did a thread dump when the issue happened. There were a total of 74 threads, so I don't think the app is running out of threads.
The dump for the Telegram thread:
"TelegramBot" #20 daemon prio=5 os_prio=0 cpu=14.65ms elapsed=47154.24s tid=0x00007f6b28e73000 nid=0x1c waiting on condition [0x00007f6aed6fb000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.13/Native Method)
- parking to wait for <0x00000000fa865c80> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(java.base@11.0.13/LockSupport.java:194)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.13/AbstractQueuedSynchronizer.java:885)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.13/AbstractQueuedSynchronizer.java:1039)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.13/AbstractQueuedSynchronizer.java:1345)
at java.util.concurrent.CountDownLatch.await(java.base@11.0.13/CountDownLatch.java:232)
at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:87)
at reactor.core.publisher.Mono.block(Mono.java:1707)
at com.moon.arbitrage.cm.feign.TelegramClient.sendMessage(TelegramClient.java:59)
at com.moon.arbitrage.cm.service.TelegramService.lambda$sendArbMessage$0(TelegramService.java:53)
at com.moon.arbitrage.cm.service.TelegramService$$Lambda$1092/0x000000084070f840.run(Unknown Source)
at com.moon.arbitrage.cm.service.TelegramService.task(TelegramService.java:82)
at com.moon.arbitrage.cm.service.TelegramService$$Lambda$920/0x0000000840665040.run(Unknown Source)
at java.lang.Thread.run(java.base@11.0.13/Thread.java:829)
Locked ownable synchronizers:
- None
The reactor worker threads:
"reactor-http-epoll-1" #15 daemon prio=5 os_prio=0 cpu=810.44ms elapsed=47157.07s tid=0x00007f6b281c4000 nid=0x17 runnable [0x00007f6b0c46c000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:177)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:286)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:351)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(java.base@11.0.13/Thread.java:829)
Locked ownable synchronizers:
- None
"reactor-http-epoll-2" #16 daemon prio=5 os_prio=0 cpu=1312.16ms elapsed=47157.07s tid=0x00007f6b281c5000 nid=0x18 waiting on condition [0x00007f6b0c369000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.13/Native Method)
- parking to wait for <0x00000000fa865948> (a java.util.concurrent.CompletableFuture$Signaller)
at java.util.concurrent.locks.LockSupport.park(java.base@11.0.13/LockSupport.java:194)
at java.util.concurrent.CompletableFuture$Signaller.block(java.base@11.0.13/CompletableFuture.java:1796)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@11.0.13/ForkJoinPool.java:3128)
at java.util.concurrent.CompletableFuture.waitingGet(java.base@11.0.13/CompletableFuture.java:1823)
at java.util.concurrent.CompletableFuture.get(java.base@11.0.13/CompletableFuture.java:1998)
at com.moon.arbitrage.cm.service.OrderService.reconcileOrder(OrderService.java:103)
at com.moon.arbitrage.cm.service.BotService$BotTask.lambda$task$1(BotService.java:383)
at com.moon.arbitrage.cm.service.BotService$BotTask$$Lambda$1161/0x00000008400af440.accept(Unknown Source)
at reactor.core.publisher.MonoPeekTerminal$MonoTerminalPeekSubscriber.onNext(MonoPeekTerminal.java:171)
at reactor.core.publisher.MonoPeekTerminal$MonoTerminalPeekSubscriber.onNext(MonoPeekTerminal.java:180)
at reactor.core.publisher.Operators$MonoSubscriber.complete(Operators.java:1816)
at reactor.core.publisher.MonoFlatMap$FlatMapInner.onNext(MonoFlatMap.java:249)
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79)
at reactor.core.publisher.FluxOnAssembly$OnAssemblySubscriber.onNext(FluxOnAssembly.java:539)
at reactor.core.publisher.Operators$MonoSubscriber.complete(Operators.java:1816)
at reactor.core.publisher.MonoFlatMap$FlatMapMain.onNext(MonoFlatMap.java:151)
at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107)
at reactor.core.publisher.FluxMapFuseable$MapFuseableConditionalSubscriber.onNext(FluxMapFuseable.java:295)
at reactor.core.publisher.FluxFilterFuseable$FilterFuseableConditionalSubscriber.onNext(FluxFilterFuseable.java:337)
at reactor.core.publisher.Operators$MonoSubscriber.complete(Operators.java:1816)
at reactor.core.publisher.MonoCollect$CollectSubscriber.onComplete(MonoCollect.java:159)
at reactor.core.publisher.FluxMap$MapSubscriber.onComplete(FluxMap.java:142)
at reactor.core.publisher.FluxPeek$PeekSubscriber.onComplete(FluxPeek.java:260)
at reactor.core.publisher.FluxMap$MapSubscriber.onComplete(FluxMap.java:142)
at reactor.netty.channel.FluxReceive.onInboundComplete(FluxReceive.java:400)
at reactor.netty.channel.ChannelOperations.onInboundComplete(ChannelOperations.java:419)
at reactor.netty.channel.ChannelOperations.terminate(ChannelOperations.java:473)
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:702)
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:93)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1372)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1235)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1284)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(java.base@11.0.13/Thread.java:829)
Locked ownable synchronizers:
- None
"reactor-http-epoll-3" #17 daemon prio=5 os_prio=0 cpu=171.84ms elapsed=47157.07s tid=0x00007f6b28beb000 nid=0x19 runnable [0x00007f6b0c26a000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:177)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:281)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:351)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(java.base@11.0.13/Thread.java:829)
Locked ownable synchronizers:
- None
"reactor-http-epoll-4" #18 daemon prio=5 os_prio=0 cpu=188.10ms elapsed=47157.07s tid=0x00007f6b28b7d800 nid=0x1a runnable [0x00007f6b0c169000]
java.lang.Thread.State: RUNNABLE
at io.netty.channel.epoll.Native.epollWait0(Native Method)
at io.netty.channel.epoll.Native.epollWait(Native.java:177)
at io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:281)
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:351)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(java.base@11.0.13/Thread.java:829)
Locked ownable synchronizers:
- None
Seems like one of them is blocked with another task (that isn't even from the Telegram service) but that shouldn't be an issue since the other three worker threads are runnable right?
ANSWER
Answered 2021-Dec-20 at 14:25I would propose to take a look in the RateLimiter direction. Maybe it does not work as expected, depending on the number of requests your application does over time. From the Javadoc for Ratelimiter: "It is important to note that the number of permits requested never affects the throttling of the request itself ... but it affects the throttling of the next request. I.e., if an expensive task arrives at an idle RateLimiter, it will be granted immediately, but it is the next request that will experience extra throttling, thus paying for the cost of the expensive task." Also helpful might be this discussion: github or github
I could imaginge there is some throttling adding up or other effect in the RateLimiter, i would try to play around with it and make sure this thing really works the way you want. Alternatively, consider using Spring @Scheduled to read from your queue. You might want to spice it up using embedded JMS for further goodies (message persistence etc).
QUESTION
I am implementing a simple chatbot using keras and WebSockets. I now have a model that can make a prediction about the user input and send the according answer.
When I do it through command line it works fine, however when I try to send the answer through my WebSocket, the WebSocket doesn't even start anymore.
Here is my working WebSocket code:
@sock.route('/api')
def echo(sock):
while True:
# get user input from browser
user_input = sock.receive()
# print user input on console
print(user_input)
# read answer from console
response = input()
# send response to browser
sock.send(response)
Here is my code to communicate with the keras model on command line:
while True:
question = input("")
ints = predict(question)
answer = response(ints, json_data)
print(answer)
Used methods are those:
def predict(sentence):
bag_of_words = convert_sentence_in_bag_of_words(sentence)
# pass bag as list and get index 0
prediction = model.predict(np.array([bag_of_words]))[0]
ERROR_THRESHOLD = 0.25
accepted_results = [[tag, probability] for tag, probability in enumerate(prediction) if probability > ERROR_THRESHOLD]
accepted_results.sort(key=lambda x: x[1], reverse=True)
output = []
for accepted_result in accepted_results:
output.append({'intent': classes[accepted_result[0]], 'probability': str(accepted_result[1])})
print(output)
return output
def response(intents, json):
tag = intents[0]['intent']
intents_as_list = json['intents']
for i in intents_as_list:
if i['tag'] == tag:
res = random.choice(i['responses'])
break
return res
So when I start the WebSocket with the working code I get this output:
* Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
* Restarting with stat
* Serving Flask app 'server' (lazy loading)
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: on
But as soon as I have anything of my model in the server.py
class I get this output:
2022-02-13 11:31:38.887640: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.
2022-02-13 11:31:38.887734: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: )
Metal device set to: Apple M1
systemMemory: 16.00 GB
maxCacheSize: 5.33 GB
It is enough when I just have an import at the top like this: from chatty import response, predict
- even though they are unused.
ANSWER
Answered 2022-Feb-16 at 19:53There is no problem with your websocket route. Could you please share how you are triggering this route? Websocket is a different protocol and I'm suspecting that you are using a HTTP client to test websocket. For example in Postman:
HTTP requests are different than websocket requests. So, you should use appropriate client to test websocket.
QUESTION
I have an ASP.Net Webforms website running in IIS on a Windows Server. Also on this server is the SQL server.
Everything has been working fine with the site but now I am seeing issues with using a DataAdapter to fill a table.
So here is some code, please note it's just basic outline of code as actual code contains confidential information.
public List
Fetch(string Connection, int parameter1, int parameter2, bool parameter3) { List collection = new List(); using (SqlConnection dbConnection = new SqlConnection(Connection)) { using (SqlCommand dbCommand = dbConnection.CreateCommand()) // Result is complex { dbCommand.CommandType = CommandType.StoredProcedure; // // Code to add parameters and set commandText goes here // using (SqlDataAdapter da = new SqlDataAdapter(dbCommand)) { DataTable table = new DataTable(); DataRow row; try { da.Fill(table); } catch(Exception ex) { // Log error } for (int i = 0; i < table.Rows.Count; i++) { row = table.Rows[i]; Summary data = Populate(row); collection.Add(data); } } } } return collection; }
The above is in a library and is called like this in the Web Forms site.
var Summaries = MyLibrary.Fetch(ConnectionString, 1, 111, false);
So as I say, everything was working fine. Now, all of a sudden the above has stopped working and Summaries
is always empty.
To investigate I tried the following.
Created a test using xUnit with the same parameters used by the website. These were captured during debugging to ensure they matched. The result returned 1 item.
I then ran the stored procedure in SQL management Studio and it matched the xUnit test with 1 item returned.
I then checked SQL Profiler and this is where things seemed a little odd. Nothing was being recorded in the trace when the web-forms was calling the library.
So both the web-site and xUnit test were using the same library, passing the same parameters and yet one worked and the other didn't... very odd.
As a last resort, I added the library project to the Web Forms project and proceeded to debug through that... then I found the error.
da.Fill(table);
The above line generated the following exception.
Exception thrown: 'Microsoft.Data.SqlClient.SqlException' in Microsoft.Data.SqlClient.dll A connection was successfully established with the server, but then an error occurred during the login process. (provider: SSL Provider, error: 0 - The certificate chain was issued by an authority that is not trusted.)
Having a look around here on stack, I saw many responses with I need to install a certificate.
My question is, if this is the case then why and why now? Nothing has changed in the code from when it was working till now. The web-site and SQL server are on the same windows server. Also, why does the xunit test work without it and the website does not when I am debugging on the same machine!
The only items that I now has changed is install of .Net 6 and patch Tuesday resulted in some updates and a restart of the server.
ANSWER
Answered 2021-Nov-27 at 15:53Microsoft.Data.SqlClient 4.0 is using ENCRYPT=True by default. Either you put a certificate on the server (not a self signed one) or you put
TrustServerCertificate=Yes;
on the connection string.
QUESTION
I'm trying to test an API endpoint with a patch request to ensure it works.
I'm using APILiveServerTestCase
but can't seem to get the permissions required to patch the item. I created one user (adminuser
) who is a superadmin with access to everything and all permissions.
My test case looks like this:
class FutureVehicleURLTest(APILiveServerTestCase):
def setUp(self):
# Setup users and some vehicle data we can query against
management.call_command("create_users_and_vehicle_data", verbosity=0)
self.user = UserFactory()
self.admin_user = User.objects.get(username="adminuser")
self.future_vehicle = f.FutureVehicleFactory(
user=self.user,
last_updated_by=self.user,
)
self.vehicle = f.VehicleFactory(
user=self.user,
created_by=self.user,
modified_by=self.user,
)
self.url = reverse("FutureVehicles-list")
self.full_url = self.live_server_url + self.url
time = str(datetime.now())
self.form_data = {
"signature": "TT",
"purchasing": True,
"confirmed_at": time,
}
I've tried this test a number of different ways - all giving the same result (403).
I have setup the python debugger in the test, and I have tried actually going to http://localhost:xxxxx/admin/
in the browser and logging in manually with any user but the page just refreshes when I click to login and I never get 'logged in' to see the admin. I'm not sure if that's because it doesn't completely work from within a debugger like that or not.
My test looks like this (using the Requests library):
def test_patch_request_updates_object(self):
data_dict = {
"signature": "TT",
"purchasing": "true",
"confirmed_at": datetime.now().strftime("%m/%d/%Y, %H:%M:%S"),
}
url = self.full_url + str(self.future_vehicle.id) + "/"
client = requests.Session()
client.auth = HTTPBasicAuth(self.admin_user.username, "test")
client.headers.update({"x-test": "true"})
response = client.get(self.live_server_url + "/admin/")
csrftoken = response.cookies["csrftoken"]
# interact with the api
response = client.patch(
url,
data=json.dumps(data_dict),
cookies=response.cookies,
headers={
"X-Requested-With": "XMLHttpRequest",
"X-CSRFTOKEN": csrftoken,
},
)
# RESPONSE GIVES 403 PERMISSION DENIED
fte_future_vehicle = FutureVehicle.objects.filter(
id=self.future_vehicle.id
).first()
# THIS ERRORS WITH '' not equal to 'TT'
self.assertEqual(fte_future_vehicle.signature, "TT")
I have tried it very similarly to the documentation using APIRequestFactory
and forcing authentication:
def test_patch_request_updates_object(self):
data_dict = {
"signature": "TT",
"purchasing": "true",
"confirmed_at": datetime.now().strftime("%m/%d/%Y, %H:%M:%S"),
}
url = self.full_url + str(self.future_vehicle.id) + "/"
api_req_factory = APIRequestFactory()
view = FutureVehicleViewSet.as_view({"patch": "partial_update"})
api_request = api_req_factory.patch(
url, json.dumps(data_dict), content_type="application/json"
)
force_authenticate(api_request, self.admin_user)
response = view(api_request, pk=self.future_assignment.id)
fte_future_assignment = FutureVehicle.objects.filter(
id=self.future_assignment.id
).first()
self.assertEqual(fte_future_assignment.signature, "TT")
If I enter the debugger to look at the responses, it's always a 403
.
The viewset
itself is very simple:
class FutureVehicleViewSet(ModelViewSet):
serializer_class = FutureVehicleSerializer
def get_queryset(self):
queryset = FutureVehicle.exclude_denied.all()
user_id = self.request.query_params.get("user_id", None)
if user_id:
queryset = queryset.filter(user_id=user_id)
return queryset
The serializer is just as basic as it gets - it's just the FutureVehicle
model and all fields.
I just can't figure out why my user won't login - or if maybe I'm doing something wrong in my attempts to patch?
I'm pretty new to Django Rest Framework in general, so any guidances is helpful!
Edit to add - my DRF Settings look like this:
REST_FRAMEWORK = {
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
"DATETIME_FORMAT": "%m/%d/%Y - %I:%M:%S %p",
"DATE_INPUT_FORMATS": ["%Y-%m-%d"],
"DEFAULT_AUTHENTICATION_CLASSES": [
# Enabling this it will require Django Session (Including CSRF)
"rest_framework.authentication.SessionAuthentication"
],
"DEFAULT_PERMISSION_CLASSES": [
# Globally only allow IsAuthenticated users access to API Endpoints
"rest_framework.permissions.IsAuthenticated"
],
}
I'm certain adminuser
is the user we wish to login - if I go into the debugger and check the users, they exist as a user. During creation, any user created has a password set to 'test'.
ANSWER
Answered 2021-Dec-11 at 07:34The test you have written is also testing the Django framework logic (ie: Django admin login). I recommend testing your own functionality, which occurs after login to the Django admin. Django's testing framework offers a helper for logging into the admin, client.login
. This allows you to focus on testing your own business logic/not need to maintain internal django authentication business logic tests, which may change release to release.
from django.test import TestCase, Client
def TestCase():
client.login(username=self.username, password=self.password)
However, if you must replicate and manage the business logic of what client.login
is doing, here's some of the business logic from Django:
def login(self, **credentials):
"""
Set the Factory to appear as if it has successfully logged into a site.
Return True if login is possible or False if the provided credentials
are incorrect.
"""
from django.contrib.auth import authenticate
user = authenticate(**credentials)
if user:
self._login(user)
return True
return False
def force_login(self, user, backend=None):
def get_backend():
from django.contrib.auth import load_backend
for backend_path in settings.AUTHENTICATION_BACKENDS:
backend = load_backend(backend_path)
if hasattr(backend, 'get_user'):
return backend_path
if backend is None:
backend = get_backend()
user.backend = backend
self._login(user, backend)
def _login(self, user, backend=None):
from django.contrib.auth import login
# Create a fake request to store login details.
request = HttpRequest()
if self.session:
request.session = self.session
else:
engine = import_module(settings.SESSION_ENGINE)
request.session = engine.SessionStore()
login(request, user, backend)
# Save the session values.
request.session.save()
# Set the cookie to represent the session.
session_cookie = settings.SESSION_COOKIE_NAME
self.cookies[session_cookie] = request.session.session_key
cookie_data = {
'max-age': None,
'path': '/',
'domain': settings.SESSION_COOKIE_DOMAIN,
'secure': settings.SESSION_COOKIE_SECURE or None,
'expires': None,
}
self.cookies[session_cookie].update(cookie_data)
Django client.login: https://github.com/django/django/blob/main/django/test/client.py#L596-L646
QUESTION
I have a df with this structure:
id a1_l1 a2_l1 a3_l1 a1_l2 a2_l2 a3_l2
1 1 5 3 1 2 3
2 1 5 3 1 2 3
3 2 5 3 5 5 3
4 5 5 3 5 5 3
5 5 5 2
6 5 5 2
7 5 5 2
8 2 5 2
9 3 5 1
10 3 5 1
I want to summarize in a table such that I get:
l1 l2
a1 0.4 0.5
a2 1 0.5
a3 0 0
In which what I'm doing is counting how may times 5 was present divided by the number of valid responses, so that for example:
a1, l1 is equal to .4 as I have 4 values of 5 divided by 10. and a2, l1 equals .5 as I have 2 values of 5 divided by 4 valid responses per column.
Thanks!
ANSWER
Answered 2021-Nov-07 at 16:25You can drop the id
column drop('id', axis=1)
as well instead of set_index('id')
- calculate the percentage for each column using
(x == 5).sum() / x.notna().sum()
- reset the index to get a column we can work with
- generate the 'a' and 'l' columns by splitting the index
- pivot
tmp = df.set_index('id').apply(lambda x: x.eq(5).sum()/x.notna().sum()).reset_index()
tmp[['a', 'l']] = tmp['index'].str.split('_', expand=True)
>>>
index 0 a l
0 a1_l1 0.4 a1 l1
1 a2_l1 1.0 a2 l1
2 a3_l1 0.0 a3 l1
3 a1_l2 0.5 a1 l2
4 a2_l2 0.5 a2 l2
5 a3_l2 0.0 a3 l2
tmp.drop('index', axis=1).pivot(index='a', columns='l').droplevel(0, axis=1).rename_axis(None).rename_axis(None, axis=1)
>>>
l1 l2
a1 0.4 0.5
a2 1.0 0.5
a3 0.0 0.0
QUESTION
I have this class:
@Configuration
@EnableWebSecurity
@EnableGlobalMethodSecurity(prePostEnabled = true)
public class ApiWebSecurityConfig extends WebSecurityConfigurerAdapter {
private static final String SALT = "fd&lkj§isfs23#$1*(_)nof";
private final JwtAuthenticationEntryPoint unauthorizedHandler;
private final JwtTokenUtil jwtTokenUtil;
private final UserSecurityService userSecurityService;
@Value("${jwt.header}")
private String tokenHeader;
public ApiWebSecurityConfig(JwtAuthenticationEntryPoint unauthorizedHandler, JwtTokenUtil jwtTokenUtil,
UserSecurityService userSecurityService) {
this.unauthorizedHandler = unauthorizedHandler;
this.jwtTokenUtil = jwtTokenUtil;
this.userSecurityService = userSecurityService;
}
@Autowired
public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception {
auth
.userDetailsService(userSecurityService)
.passwordEncoder(passwordEncoder());
}
@Bean
public BCryptPasswordEncoder passwordEncoder() {
return new BCryptPasswordEncoder(12, new SecureRandom(SALT.getBytes()));
}
@Bean
@Override
public AuthenticationManager authenticationManagerBean() throws Exception {
return super.authenticationManagerBean();
}
@Override
protected void configure(HttpSecurity httpSecurity) throws Exception {
httpSecurity
// we don't need CSRF because our token is invulnerable
.csrf().disable()
.exceptionHandling().authenticationEntryPoint(unauthorizedHandler).and()
// don't create session
.sessionManagement().sessionCreationPolicy(SessionCreationPolicy.STATELESS).and()
.authorizeRequests()
// Un-secure H2 Database
.antMatchers("/h2-console/**/**").permitAll()
.antMatchers("/api/v1/users").permitAll()
.antMatchers("/error").permitAll()
.anyRequest().authenticated();
// Custom JWT based security filter
JwtAuthorizationTokenFilter authenticationTokenFilter = new JwtAuthorizationTokenFilter(userDetailsService(), jwtTokenUtil, tokenHeader);
httpSecurity
.addFilterBefore(authenticationTokenFilter, UsernamePasswordAuthenticationFilter.class);
// disable page caching
httpSecurity
.headers()
.frameOptions()
.sameOrigin() // required to set for H2 else H2 Console will be blank.
.cacheControl();
}
@Override
public void configure(WebSecurity web) {
// AuthenticationTokenFilter will ignore the below paths
web
.ignoring()
.antMatchers(
HttpMethod.POST,
"/api/v1/auth"
)
.antMatchers(
HttpMethod.POST,
"/api/v1/users"
)
.antMatchers(
HttpMethod.GET,
"/api/v1/countries"
);
}
}
and
@Provider
@Slf4j
public class JwtAuthorizationTokenFilter extends OncePerRequestFilter {
private UserDetailsService userDetailsService;
private JwtTokenUtil jwtTokenUtil;
private String tokenHeader;
public JwtAuthorizationTokenFilter(UserDetailsService userDetailsService,
JwtTokenUtil jwtTokenUtil,
String tokenHeader) {
this.userDetailsService = userDetailsService;
this.jwtTokenUtil = jwtTokenUtil;
this.tokenHeader = tokenHeader;
}
@Override
protected boolean shouldNotFilter(HttpServletRequest request) {
return new AntPathMatcher().match("/api/v1/users",
request.getServletPath());
}
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws ServletException,
IOException {
log.info("processing authentication for '{}'", request.getRequestURL());
log.info("tokenHeader '{}'", tokenHeader);
final String requestHeader = request.getHeader(this.tokenHeader);
log.info("requestHeader '{}'", requestHeader);
String username = null;
String authToken = null;
if (requestHeader != null && requestHeader.startsWith("Bearer ")) {
authToken = requestHeader.substring(7);
log.info("authToken '{}'", authToken);
try {
username = jwtTokenUtil.getUsernameFromToken(authToken);
} catch (IllegalArgumentException e) {
logger.info("an error occured during getting username from token", e);
} catch (ExpiredJwtException e) {
logger.info("the token is expired and not valid anymore", e);
}
} else {
logger.info("couldn't find bearer string, will ignore the header");
}
log.info("checking authentication for user '{}'", username);
if (username != null && SecurityContextHolder.getContext().getAuthentication() == null) {
logger.info("security context was null, so authorizating user");
// It is not compelling necessary to load the use details from the database. You could also store the information
// in the token and read it from it. It's up to you ;)
UserDetails userDetails = this.userDetailsService.loadUserByUsername(username);
// For simple validation it is completely sufficient to just check the token integrity. You don't have to call
// the database compellingly. Again it's up to you ;)
if (jwtTokenUtil.validateToken(authToken, userDetails)) {
UsernamePasswordAuthenticationToken authentication = new UsernamePasswordAuthenticationToken(userDetails, null, userDetails.getAuthorities());
authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(request));
log.info("authorizated user '{}', setting security context", username);
SecurityContextHolder.getContext().setAuthentication(authentication);
}
}
chain.doFilter(request, response);
}
}
and
@RestController
@Slf4j
public class AuthenticationRestController {
private static final Logger LOG = LoggerFactory.getLogger (AuthenticationRestController.class);
@Value("${jwt.header}")
private String tokenHeader;
@Autowired
private AuthenticationManager authenticationManager;
@Autowired
private JwtTokenUtil jwtTokenUtil;
@Autowired
private UserSecurityService userSecurityService;
@PostMapping(path = "/api/v1/auth", consumes = "application/json", produces = "application/json")
public ResponseEntity
createAuthenticationToken( @RequestBody JwtAuthenticationRequest authenticationRequest,
HttpServletRequest request) throws AuthenticationException {
LOG.info("authenticating {} " , authenticationRequest.getUsername());
authenticate(authenticationRequest.getUsername(), authenticationRequest.getPassword());
// Reload password post-security so we can generate the token
final User userDetails = (User) userSecurityService.loadUserByUsername(authenticationRequest.getUsername());
if (!userDetails.isEnabled()) {
throw new UserDisabledException();
}
if (LOG.isDebugEnabled()) {
LOG.debug("UserDetails userDetails [ " + authenticationRequest.getUsername() + " ]");
}
final String token = jwtTokenUtil.generateToken(userDetails);
// Return the token
return ResponseEntity.ok(new JwtAuthenticationResponse(token));
}
@GetMapping(path = "${jwt.route.authentication.refresh}", consumes = "application/json", produces = "application/json")
public ResponseEntity refreshAndGetAuthenticationToken(HttpServletRequest request) {
String authToken = request.getHeader(tokenHeader);
final String token = authToken.substring(7);
String username = jwtTokenUtil.getUsernameFromToken(token);
JwtUser user = (JwtUser) userSecurityService.loadUserByUsername(username);
if (jwtTokenUtil.canTokenBeRefreshed(token, user.getLastPasswordResetDate())) {
String refreshedToken = jwtTokenUtil.refreshToken(token);
return ResponseEntity.ok(new JwtAuthenticationResponse(refreshedToken));
} else {
return ResponseEntity.badRequest().body(null);
}
}
@ExceptionHandler({AuthenticationException.class})
public ResponseEntity handleAuthenticationException(AuthenticationException e) {
return ResponseEntity.status(HttpStatus.UNAUTHORIZED).body(e.getMessage());
}
/**
* Authenticates the user. If something is wrong, an {@link AuthenticationException} will be thrown
*/
private void authenticate(String username, String password) {
Objects.requireNonNull(username);
Objects.requireNonNull(password);
try {
authenticationManager.authenticate(new UsernamePasswordAuthenticationToken(username, password));
} catch (DisabledException e) {
e.printStackTrace();
throw new AuthenticationException("User is disabled!", e);
} catch (BadCredentialsException e) {
throw new AuthenticationException("Bad credentials!", e);
} catch (Exception e) {
e.printStackTrace();
}
}
}
and
@RestController
@RequestMapping("/api/v1/styles")
@Slf4j
public class StyleResourceController {
@PutMapping(path = "/{styleCode}")
@ResponseStatus(HttpStatus.OK)
public void setAlerts(@RequestHeader(value = "Authorization") String authHeader, @PathVariable String styleCode)
throws DataAccessException {
System.out.println("add style {} ");
final User user = authUserOnPath(authHeader);
System.out.println("user {} " + user);
protected User authUserOnPath(String authHeader) {
String authToken = authHeader.substring(7);
String username = jwtTokenUtil.getUsernameFromToken(authToken);
User user = userService.findByUserName(username);
if (user == null)
throw new ResponseStatusException(HttpStatus.UNAUTHORIZED, "UserNotFound");
return user;
}
}
}
and
@Component
public class JwtTokenUtil implements Serializable {
//static final String CLAIM_KEY_USERNAME = "sub";
//static final String CLAIM_KEY_CREATED = "iat";
private static final long serialVersionUID = -3301605591108950415L;
// @SuppressFBWarnings(value = "SE_BAD_FIELD", justification = "It's okay here")
private Clock clock = DefaultClock.INSTANCE;
@Value("${jwt.secret}")
private String secret;
@Value("${jwt.expiration}")
private Long expiration;
public String getUsernameFromToken(String token) {
return getClaimFromToken(token, Claims::getSubject);
}
public Date getIssuedAtDateFromToken(String token) {
return getClaimFromToken(token, Claims::getIssuedAt);
}
public Date getExpirationDateFromToken(String token) {
return getClaimFromToken(token, Claims::getExpiration);
}
public T getClaimFromToken(String token, Function claimsResolver) {
final Claims claims = getAllClaimsFromToken(token);
return claimsResolver.apply(claims);
}
private Claims getAllClaimsFromToken(String token) {
return Jwts.parser()
.setSigningKey(secret)
.parseClaimsJws(token)
.getBody();
}
private Boolean isTokenExpired(String token) {
final Date expiration = getExpirationDateFromToken(token);
return expiration.before(clock.now());
}
private Boolean isCreatedBeforeLastPasswordReset(Date created, Date lastPasswordReset) {
return (lastPasswordReset != null && created.before(lastPasswordReset));
}
private Boolean ignoreTokenExpiration(String token) {
// here you specify tokens, for that the expiration is ignored
return false;
}
public String generateToken(UserDetails userDetails) {
Map claims = new HashMap<>();
return doGenerateToken(claims, userDetails.getUsername());
}
private String doGenerateToken(Map claims, String subject) {
final Date createdDate = clock.now();
final Date expirationDate = calculateExpirationDate(createdDate);
return Jwts.builder()
.setClaims(claims)
.setSubject(subject)
.setIssuedAt(createdDate)
.setExpiration(expirationDate)
.signWith(SignatureAlgorithm.HS512, secret)
.compact();
}
public Boolean canTokenBeRefreshed(String token, Date lastPasswordReset) {
final Date created = getIssuedAtDateFromToken(token);
return !isCreatedBeforeLastPasswordReset(created, lastPasswordReset)
&& (!isTokenExpired(token) || ignoreTokenExpiration(token));
}
public String refreshToken(String token) {
final Date createdDate = clock.now();
final Date expirationDate = calculateExpirationDate(createdDate);
final Claims claims = getAllClaimsFromToken(token);
claims.setIssuedAt(createdDate);
claims.setExpiration(expirationDate);
return Jwts.builder()
.setClaims(claims)
.signWith(SignatureAlgorithm.HS512, secret)
.compact();
}
public Boolean validateToken(String token, UserDetails userDetails) {
JwtUser user = (JwtUser) userDetails;
final String username = getUsernameFromToken(token);
final Date created = getIssuedAtDateFromToken(token);
return (
username.equals(user.getUsername())
&& !isTokenExpired(token)
&& !isCreatedBeforeLastPasswordReset(created, user.getLastPasswordResetDate())
);
}
private Date calculateExpirationDate(Date createdDate) {
return new Date(createdDate.getTime() + expiration * 1000);
}
}
but when I get the user from the token is null:
17:19:22.017 [http-nio-1133-exec-8] INFO c.d.c.JwtAuthorizationTokenFilter - tokenHeader 'Authorization'
17:19:22.017 [http-nio-1133-exec-8] INFO c.d.c.JwtAuthorizationTokenFilter - requestHeader 'Bearer eyJhbGciOiJIUzUxMiJ9.eyJleHAiOjE2OTE3NjcxMzYsImlhdCI6MTYzMTI4NzEzNn0.C9s3dbjWNVyGdV5k0LXsNhMGMvPzboTx1J6sGEbfXVOP1CzCLeZFgVPQ4o8jgugvgURF3BcnsWAk7ygd7RCvdg'
17:19:22.017 [http-nio-1133-exec-8] INFO c.d.c.JwtAuthorizationTokenFilter - authToken 'eyJhbGciOiJIUzUxMiJ9.eyJleHAiOjE2OTE3NjcxMzYsImlhdCI6MTYzMTI4NzEzNn0.C9s3dbjWNVyGdV5k0LXsNhMGMvPzboTx1J6sGEbfXVOP1CzCLeZFgVPQ4o8jgugvgURF3BcnsWAk7ygd7RCvdg'
17:19:22.018 [http-nio-1133-exec-8] INFO c.d.c.JwtAuthorizationTokenFilter - checking authentication for user 'null'
ANSWER
Answered 2021-Sep-18 at 07:25Double check your jwt token. I think it miss sub attribute( subject or username here).
I also highly recommend you write the few unit test for few class such as JwtTokenUtil to make sure your code working as expected. You can use spring-test to do it easily.
It help you discover the bug easier and sooner.
Here is few test which i used to test the commands "jwt generate" and "jwt parse"
@Test
public void testInvalidKey() {
String result = execute("jwt generate ");
Assertions.assertEquals(JsonWebToken.ERR_LOAD_PRIVATE_KEY, result);
}
@Test
public void testInvalidExpiredDate() {
String result = execute(
"jwt generate -exp \"12-12-2020\"");
Assertions.assertEquals(JsonWebToken.ERR_INVALID_EXPIRED_DATE, result);
}
@Test
public void testGenerateOk() {
String result = execute(
"jwt generate");
Assertions.assertNotNull(result);
Assertions.assertTrue(result.length() > 100);
}
@Test
public void testParseToken() {
String result = execute(
"jwt generate -sub designer");
Assertions.assertNotNull(result);
String parse = execute("jwt parse -token " + result);
Assertions.assertTrue(parse.contains("sub:designer"));
}
And here is few other test which i used to check the controller and filter.
@Test
public void testInValidToken() throws Exception{
Map map = new LinkedHashMap<>(reqObj);
mockMvc.perform(postWithTokenAndData(getUrlProvider(), map)
.header(HEADER_AUTHORIZATION, String.format("Bearer %s", INVALID_JWT))
.contentType(MediaType.APPLICATION_JSON))
.andExpect(MockMvcResultMatchers.status().is4xxClientError());
}
@Test
public void testExpired() throws Exception{
Date exp = Date.from(Instant.now().minusSeconds(3600));
Map map = new LinkedHashMap<>(reqObj);
mockMvc.perform(postWithTokenAndData(getUrlProvider(), map)
.header(HEADER_AUTHORIZATION, String.format("Bearer %s", JwtHelperTest.getJwtToken(exp,
"designer")))
.contentType(MediaType.APPLICATION_JSON))
.andExpect(MockMvcResultMatchers.status().is4xxClientError());
}
QUESTION
EDIT
I'm trying to import algosec.models
in a file inside the algobot
package.
I've tried to add --hidden-import algosec
, I've also tried to add the path before importing, using sys.path.append(./../algosec)
this is the error message I get when I try to run the program:
Traceback (most recent call last):
File "algobot_packer/algobot.py", line 2, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/cli/cli.py", line 3, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/microsoft_teams/mainloop.py", line 9, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/framework/configuration.py", line 34, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/framework/commands.py", line 22, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/framework/bot.py", line 4, in
File "PyInstaller/loader/pyimod03_importers.py", line 546, in exec_module
File "algobot/framework/responses.py", line 9, in
ModuleNotFoundError: No module named 'algosec'
the folder structure is:
- algobot
- algobot
- algosec
- algobot-packer
- pyucwa
I'm using pyinstaller
version 4.2
I didn't make any change in the code since the last time my executable file ran perfectly fine, but now I'm getting this error every time. the thing is - the folder 'algosec' is a subdirectory in my project, and it is noted in the pipfile and again, I didn't make any change in a while and tested it recently (last tested on July 8th)), therefore I believe that it's a dependency issue but not sure which or how to solve.
I've tried multiple changes that somehow worked on one run but when I tried to make these changes again it failed on other builds...
ANSWER
Answered 2021-Aug-31 at 06:13You can use
--add-data "path_to_algobot:."
based on your system for windows
use ;
and for linux
use :
It will explicitly add your algosec folder into the package.
QUESTION
Say we have n servers, and we only need k < n responses .
I understand futures::join_all can be used to wait for all n futures, but I want my program finish waiting after k responses.
Is there something similar to join_all that I can use to wait for the first k responses?
ANSWER
Answered 2021-Jul-20 at 04:17I don't believe there is anything built for this purpose. Perhaps you can do this with Stream
s or Channel
s, but the join_all
implementation isn't too complicated. I've modified it so that it only waits for n
results:
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use futures::future::MaybeDone; // 0.3.15
fn iter_pin_mut(slice: Pin<&mut [T]>) -> impl Iterator> {
// Safety: `std` _could_ make this unsound if it were to decide Pin's
// invariants aren't required to transmit through slices. Otherwise this has
// the same safety as a normal field pin projection.
//
// Copied from `futures` implementation of `join_all`.
unsafe { slice.get_unchecked_mut() }.iter_mut().map(|t| unsafe { Pin::new_unchecked(t) })
}
#[must_use = "futures do nothing unless you `.await` or poll them"]
pub struct JoinSome
where
F: Future,
{
elems: Pin<[MaybeDone]>>,
count: usize,
}
/// Will wait for at least `n` futures to complete. More may be returned if
/// multiple resolve around the same time.
///
/// # Panics
///
/// Will panic if iterator doesn't contain at least `n` futures.
pub fn join_some(i: I, n: usize) -> JoinSome
where
I: IntoIterator,
I::Item: Future,
{
let elems: Box<[_]> = i.into_iter().map(MaybeDone::Future).collect();
assert!(elems.len() >= n);
JoinSome { elems: elems.into(), count: n }
}
impl Future for JoinSome
where
F: Future,
{
type Output = Vec;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll {
let mut num_done = 0;
for elem in iter_pin_mut(self.elems.as_mut()) {
if !elem.poll(cx).is_pending() {
num_done += 1;
}
}
if num_done >= self.count {
let mut elems = std::mem::replace(&mut self.elems, Box::pin([]));
let result = iter_pin_mut(elems.as_mut()).filter_map(|e| e.take_output()).collect();
Poll::Ready(result)
} else {
Poll::Pending
}
}
}
I've added documentation in the source that explains its behavior. You can see that it works with this simple test program:
#[tokio::main]
async fn main() {
use std::time::{Instant, Duration};
use tokio::time::sleep;
let futures = vec![
sleep(Duration::from_secs(1)),
sleep(Duration::from_secs(2)),
sleep(Duration::from_secs(3)),
sleep(Duration::from_secs(4)),
sleep(Duration::from_secs(5)),
];
let now = Instant::now();
let some = join_some(futures, 3).await;
let elapsed = now.elapsed();
println!("{} results in {:.2?}", some.len(), elapsed);
}
3 results in 3.00s seconds
See it working on the playground.
Community Discussions, Code Snippets contain sources that include Stack Exchange Network
Vulnerabilities
No vulnerabilities reported
Install responses
You can use responses like any standard Python library. You will need to make sure that you have a development environment consisting of a Python distribution including header files, a compiler, pip, and git installed. Make sure that your pip, setuptools, and wheel are up to date. When using pip it is generally recommended to install packages in a virtual environment to avoid changes to the system.
Support
Find, review, and download reusable Libraries, Code Snippets, Cloud APIs from over 650 million Knowledge Items
Find more librariesExplore Kits - Develop, implement, customize Projects, Custom Functions and Applications with kandi kits
Save this library and start creating your kit
Share this Page