Pickle data was truncated ошибка

I can not send my numpy array in socket. I use pickle but my client pickle crashes with this error: pickle data was truncated

My server :
I create a numpy array and I want to send in my client with pickle (it’s work)

import socket, pickle
import numpy as np
from PIL import ImageGrab
import cv2


while(True):
    HOST = 'localhost'
    PORT = 50007
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096)
    s.bind((HOST, PORT))
    s.listen(1)
    conn, addr = s.accept()
    print ('Connected by', addr)

    arr = np.array([[0, 1], [2, 3]])
    printscreen_pil=ImageGrab.grab(bbox=(10,10,500,500))
    img = np.array(printscreen_pil) ## Transform to Array
    
    data_string = pickle.dumps(img)
    conn.send(data_string)

    msg_recu = conn.recv(4096)
    print(msg_recu.decode())

    conn.close()

My client
He has my numpy array, but I can not load with pickle. I have this error.

import socket, pickle
import numpy as np

HOST = 'localhost'
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))

msg_a_envoyer = "hello".encode()
s.send(msg_a_envoyer)


while 1:
    data = s.recv(4096)
    if not data: break
    data_arr = pickle.loads(data)
    print (data_arr)
s.close()

🐛 Bug

I am trying to load my monodatasets for XLM and am stumped with this pickle data issue. I have attempted to pass various arguments including expression ascii, latin1 and utf-8
data = torch.load(path) File "libsite-packagestorchserialization.py", line 358, in load return _load(f, map_location, pickle_module) File "libsite-packagestorchserialization.py", line 532, in _load magic_number = pickle_module.load(f) _pickle.UnpicklingError: pickle data was truncated

To Reproduce

I am working with 0.4 pytorch on the recent (translation model)[https://github.com/facebookresearch/XLM/]

Environment

PyTorch version: N/A
Is debug build: N/A
CUDA used to build PyTorch: N/A

OS: Microsoft Windows 10 Pro
GCC version: Could not collect
CMake version: Could not collect

Python version: 3.5
Is CUDA available: N/A
CUDA runtime version: 9.0
GPU models and configuration: GPU 0: GeForce GTX 960M
Nvidia driver version: 419.35
cuDNN version: C:Program FilesNVIDIA GPU Computing ToolkitCUDAv9.0bincudnn64_7.dll

Versions of relevant libraries:
[pip] numpy==1.16.2
[pip] torch==0.4.1
[conda] blas 1.0 mkl
[conda] cuda90 1.0 0 pytorch
[conda] mkl 2019.1 144
[conda] mkl 2019.0
[conda] mkl_fft 1.0.10 py36h14836fe_0
[conda] mkl_random 1.0.2 py36h343c172_0
[conda] pytorch 0.4.1 py36_cuda90_cudnn7he774522_1 pytorch

Additional context

Issue

While trying to send a list to a client process, I pickle it, but when I receive this on the client side it always gives me the error pickle data was truncated, and I don’t know how to fix it.

sv

def handle_client(connection):
    connection.send(str.encode('welcome to sv'))
    stock = random.sample(output, 1)
    order = pickle.dumps(stock)
    while True:
        data = connection.recv(2048)
        if not data:
            break
    if data.decode('utf-8') == 'wanna trade!':
        print("trade order received")
        tcp.send(order)
    reply = connection.recv(2048)
    if reply.decode('utf-8') == 'Y':
       tcp.send(order)
       output.remove(order)
    elif reply.decode('utf-8') == 'N':
        print("doesn't wish to buy.")
    connection.close()

client

while True:
    Cliente.send(str.encode('wanna trade!'))
    recv = Cliente.recv(2048)
    if not recv:
        break
    if recv:
        Response = pickle.loads(recv)
        print(Response)
        Check = input('Y/N: ')
        Cliente.send(str.encode(Check))
    recv2 = Cliente.recv(2048)
    if recv2:
        final = pickle.load(recv2)
        purchases.append(final.decode('utf-8'))
        print(purchases)
Cliente.close()

Solution

I can’t test it but data can be longer than 2048 bytes and when you get from socket only 2048 bytes then you get truncated data.

Socket doesn’t know how many data to get in client so you have to send size before data. And size should have always the same length (so client will know if it get full size) so sending it as string may not work (or you would have to read size char after char until you get some spcial char – ie. new line which server would have to send after string with size.)

Server should first send data size (as ie. 4 bytes converted with struct.pack()) and later send data.

And client should first read 4 bytes and convert it to integer with struct.unpack() and later use this value to read all data.

Server:

import struct

stock = ...
data = pickle.dumps(stock)

size = len(data)
size_in_4_bytes = struct.pack('I', size)

print(size, size_in_4_bytes)

tcp.send(size_in_4_bytes)
tcp.send(data)

Client:

import struct

size_in_4_bytes = Cliente.recv(4)  # get only 4 bytes
size = struct.unpack('I', size_in_4_bytes)
size = size[0]

print(size, size_in_4_bytes)

data = Cliente.recv(size)
stock = pickle.loads(data)

EDIT:

If you put code in functions then you could use it many times in simply way. You could use it also to send different object: pickle, normal string, data as JSON string, image, any file.

import struct

def send_data(conn, data):
    size = len(data)
    size_in_4_bytes = struct.pack('I', size)
    conn.send(size_in_4_bytes)
    conn.send(data)

def recv_data(conn):
    size_in_4_bytes = conn.recv(4)
    size = struct.unpack('I', size_in_4_bytes)
    size = size[0]
    data = conn.recv(size)
    return data

# -------------------------------------

# --- pickle ---

# send pickle
data = pickle.dumps(stock)
send_data(Client, data)

# recv pickle
data = recv_data(Client)
stock = pickle.loads(data)

# --- text ---

# send normal string
data = text.encode()
send_data(Client, data)

# recv normal string
data = recv_data(Client)
text = data.decode()

# --- JSON ---

# send data as JSON
stock = {'open': 12, 'close': 15}
text = json.dumps(stock)
data = text.encode()
send_data(Client, data)

# recv data as JSON
data = recv_data(Client)
text = data.decode()
stock = json.loads(text)
print(stock) # {'open': 12, 'close': 15}

# --- image (or any other file) ---

# send image
with open('image.jpg', 'rb') as image
    data = image.read()
    send_data(Client, data)

# recv image
with open('image.jpg', 'wb') as image
    data = recv_data(Client)
    image.write(data)

EDIT:

Full working example.

Client first sends text and receives text, next it sends directory converted to JSON, and it receives JSON with other directory.

Server uses threads to run with many clients at the same time. There is sleep() to have time to start another client.

I use my code from answer for question:
How to handle multithreading with sockets in Python?

Server:

import socket
import threading
import time
import struct
import json

# --- functions ---

def send_data(conn, data):
    size = len(data)
    size_in_4_bytes = struct.pack('I', size)
    conn.send(size_in_4_bytes)
    conn.send(data)

def recv_data(conn):
    size_in_4_bytes = conn.recv(4)
    size = struct.unpack('I', size_in_4_bytes)
    size = size[0]
    data = conn.recv(size)
    return data

def handle_client(conn, addr):
    print("[thread] starting")

    # ---
    
    # recv message
    
    data = recv_data(conn)
    text = data.decode()

    print("[thread] client:", addr, 'recv:', text)
    
    # simulate longer work - to start next client at the same time
    time.sleep(5) 

    # send message
    
    text = "Bye!"
    print("[thread] client:", addr, 'send:', text)

    data = text.encode()
    send_data(conn, data)
    
    # ---

    # recv JSON

    data = recv_data(conn)
    text = data.decode()
    stock = json.loads(text)

    print("[thread] client:", addr, 'recv:', stock)

    # send JSON

    stock = {'diff': stock['close'] - stock['open']}
    
    print("[thread] client:", addr, 'send:', stock)

    text = json.dumps(stock)
    data = text.encode()
    send_data(conn, data)
    
    # ---
    
    conn.close()

    print("[thread] ending")
   
# --- main ---


host = '0.0.0.0'
port = 8080

s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)  # solution for "[Error 89] Address already in use". Use before bind()
s.bind((host, port))
s.listen(1)

all_threads = []

try:
    while True:
        print("Waiting for client")
        conn, addr = s.accept()
    
        print("Client:", addr)
        
        t = threading.Thread(target=handle_client, args=(conn, addr))
        t.start()
    
        all_threads.append(t)
except KeyboardInterrupt:
    print("Stopped by Ctrl+C")
finally:
    if s:
        s.close()
    for t in all_threads:
        t.join()
    

Client:

import socket
import struct
import json

# --- functions ---

def send_data(conn, data):
    size = len(data)
    size_in_4_bytes = struct.pack('I', size)
    conn.send(size_in_4_bytes)
    conn.send(data)

def recv_data(conn):
    size_in_4_bytes = conn.recv(4)
    size = struct.unpack('I', size_in_4_bytes)
    size = size[0]
    data = conn.recv(size)
    return data

# --- main ---

host = '0.0.0.0'
port = 8080

s = socket.socket()
s.connect((host, port))

print("Connected to the server")

# ---

# send message

text = "Hello"

print('send:', text)

data = text.encode()
send_data(s, data)

# recv message

data = recv_data(s)
text = data.decode()

print('recv:', text)

# ---

# send JSON

stock = {'open': 12, 'close': 15}

print('send:', stock)

text = json.dumps(stock)
data = text.encode()
send_data(s, data)

# recv JSON

data = recv_data(s)
text = data.decode()
stock = json.loads(text)

print('recv:', stock)

# ---

s.close()

Similar way client could send filename and server could send back image data. But for files it may need receiving in chunks because socket has limited buffer. It may need also to send extra iformation if server found image or not.

Answered By – furas

This Answer collected from stackoverflow, is licensed under cc by-sa 2.5 , cc by-sa 3.0 and cc by-sa 4.0