Skip to content

bpo-32410: Avoid blocking on file IO in sendfile fallback code #7172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
1st1 merged 2 commits into from
May 28, 2018
Merged

bpo-32410: Avoid blocking on file IO in sendfile fallback code #7172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
66bfe63
bpo-32410: Avoid blocking on file IO in sendfile fallback code
1st1 May 28, 2018
c6a2a85
Fix tests on win
1st1 May 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions Lib/asyncio/base_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,7 +800,10 @@ async def _sock_sendfile_native(self, sock, file, offset, count):
async def _sock_sendfile_fallback(self, sock, file, offset, count):
if offset:
file.seek(offset)
blocksize = min(count, 16384) if count else 16384
blocksize = (
min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
)
buf = bytearray(blocksize)
total_sent = 0
try:
Expand All @@ -810,7 +813,7 @@ async def _sock_sendfile_fallback(self, sock, file, offset, count):
if blocksize <= 0:
break
view = memoryview(buf)[:blocksize]
read = file.readinto(view)
read = await self.run_in_executor(None, file.readinto, view)
if not read:
break # EOF
await self.sock_sendall(sock, view)
Expand Down
4 changes: 4 additions & 0 deletions Lib/asyncio/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
# Number of seconds to wait for SSL handshake to complete
SSL_HANDSHAKE_TIMEOUT = 10.0

# Used in sendfile fallback code. We use fallback for platforms
# that don't support sendfile, or for TLS connections.
SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256

# The enum should be here to break circular dependencies between
# base_events and sslproto
class _SendfileMode(enum.Enum):
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_asyncio/test_base_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1818,12 +1818,15 @@ async def wait_closed(self):

@classmethod
def setUpClass(cls):
cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE
constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16
with open(support.TESTFN, 'wb') as fp:
fp.write(cls.DATA)
super().setUpClass()

@classmethod
def tearDownClass(cls):
constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
support.unlink(support.TESTFN)
super().tearDownClass()

Expand Down
11 changes: 11 additions & 0 deletions Lib/test/test_asyncio/test_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2142,6 +2142,17 @@ def connection_lost(self, exc):
async def wait_closed(self):
await self.fut

@classmethod
def setUpClass(cls):
cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE
constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16
super().setUpClass()

@classmethod
def tearDownClass(cls):
constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
super().tearDownClass()

def set_socket_opts(self, sock):
# On macOS, SO_SNDBUF is reset by connect(). So this method
# should be called after the socket is connected.
Expand Down
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Avoid blocking on file IO in sendfile fallback code