Own prediction problem: Extreme memory use, any solution?

from collections import defaultdict
from surprise import Dataset, Reader
from surprise.similarities import \
    cosine, msd, pearson, pearson_baseline
from surprise.prediction_algorithms.knns import \
   KNNBasic, KNNWithMeans, KNNWithZScore, KNNBaseline
from surprise.model_selection import \
    train_test_split, GridSearchCV, cross_validate
from surprise import accuracy, SVD
from surprise.model_selection import KFold
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import numpy as np
import csv
from surprise import accuracy
from datetime import datetime
start_time = datetime.now()

def get_top_n(predictions, n=10):
    """Return the top-N recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        n(int): The number of recommendation to output for each user. Default
            is 10.

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    """

    # First map the predictions to each user.
    top_n = defaultdict(list)
    for uid, iid, true_r, est, _ in predictions:
        top_n[uid].append((iid, est))

    # Then sort the predictions for each user and retrieve the k highest ones.
    for uid, user_ratings in top_n.items():
        user_ratings.sort(key=lambda x: x[1], reverse=False)
        top_n[uid] = user_ratings[:n]

    return top_n


# First train an SVD algorithm 
postulaciones ='c1_postulaciones.csv'
reader = Reader(rating_scale = (1,12))
data = Dataset.load_from_df(data[["mrun", "rbd", "preferencia_postulante"]], reader)
trainset = data.build_full_trainset()
algo = SVD()
algo.fit(trainset)

# Than predict ratings for all pairs (u, i) that are NOT in the training set.
testset = trainset.build_anti_testset()
predictions = algo.test(testset)

top_n = get_top_n(predictions, n=1)

##Dataframe con recomendaciones##
columnas = ["mrun", "rbd"]
recomendaciones = pd.DataFrame(columns = columnas) 
# Print the recommended items for each user
for uid, user_ratings in top_n.items():
    recomendacion=[iid for (iid, _) in user_ratings]
    print(uid, [iid for (iid, _) in user_ratings])
    agregar = {'mrun': uid, 'rbd': recomendacion}
    recomendaciones = recomendaciones.append(agregar, ignore_index=True)
    print("Trabajando")
recomendaciones['rbd'] = recomendaciones['rbd'].str[0]
recomendaciones.to_csv('recomendaciones.csv',sep=";")