SVM дал плохой результат в моих данных. Как исправить?

Вот аналогичный подход. Мы разделим наборы данных на train и test. Набор данных train будет использоваться для настройки гиперпараметров и подбора различных моделей. Затем мы выберем лучшую (с точки зрения MSE) модель и предскажем значения из набора данных test.

Все обученные (подогнанные) модели будут сохранены в виде файлов Pickle, поэтому их можно будет загрузить позже, используя метод joblib.load().

Вывод:

----------------------------- [SVR_rbf] ------------------------------
Fitting 3 folds for each of 4 candidates, totalling 12 fits
---------------------------- [SVR_linear] ----------------------------
Fitting 3 folds for each of 4 candidates, totalling 12 fits
------------------------------ [Ridge] -------------------------------
Fitting 3 folds for each of 7 candidates, totalling 21 fits
------------------------------ [Lasso] -------------------------------
Fitting 3 folds for each of 6 candidates, totalling 18 fits
--------------------------- [RandomForest] ---------------------------
Fitting 3 folds for each of 3 candidates, totalling 9 fits
----------------------------- [SVR_rbf] ------------------------------
Score:      44.88%
Parameters: {'SVR_rbf__C': 10, 'SVR_rbf__max_iter': 500}
**********************************************************************
---------------------------- [SVR_linear] ----------------------------
Score:      33.40%
Parameters: {'SVR_linear__C': 0.01, 'SVR_linear__max_iter': 1000}
**********************************************************************
------------------------------ [Ridge] -------------------------------
Score:      34.83%
Parameters: {'Ridge__alpha': 500, 'Ridge__max_iter': 200}
**********************************************************************
------------------------------ [Lasso] -------------------------------
Score:      22.90%
Parameters: {'Lasso__alpha': 0.1, 'Lasso__max_iter': 1000}
**********************************************************************
--------------------------- [RandomForest] ---------------------------
Score:      36.87%
Parameters: {'RandomForest__max_depth': 5, 'RandomForest__n_estimators': 250}
**********************************************************************
Mean Squared Error: {'SVR_rbf': 5.375, 'SVR_linear': 7.036, 'Ridge': 7.02, 'Lasso': 8.108, 'RandomForest': 9.475}

Код:

import os
#import contextlib
from operator import itemgetter
from pathlib import Path
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import SGDRegressor, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVR
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.externals import joblib


def get_data(path='.'):
    p = Path(path)
    kwargs = dict(delim_whitespace=True, header=None)
    X_train = pd.read_csv(list(p.glob('trainX.txt*'))[0], **kwargs)
    y_train = pd.read_csv(list(p.glob('trainY.txt*'))[0], **kwargs)
    X_test = pd.read_csv(list(p.glob('testX.txt*'))[0], **kwargs)
    y_test = pd.read_csv(list(p.glob('testY.txt*'))[0], **kwargs)
    return (pd.concat([X_train, X_test], ignore_index=True),
            pd.concat([y_train, y_test], ignore_index=True)[0])


def get_data_split(path='.', test_size=0.25):
    X, y = get_data(path)
    return train_test_split(X, y, test_size=test_size)


def tune_models_hyperparams(X, y, models, **common_grid_kwargs):
    grids = {}
    for model in models:
        print('{:-^70}'.format(' [' + model['name'] + '] '))
        pipe = Pipeline([
                    ("scale", StandardScaler()),
                    (model['name'], model['model'])   ])
        grids[model['name']] = (GridSearchCV(pipe,
                                           param_grid=model['param_grid'],
                                           **common_grid_kwargs)
                                  .fit(X, y))
        # saving single trained model ...
        joblib.dump(grids[model['name']], './{}.pkl'.format(model['name']))
    return grids


def get_best_model(grid, X_test, y_test,
                        metric_func=mean_squared_error):
    res = {name : round(metric_func(y_test, model.predict(X_test)), 3)
           for name, model in grid.items()}
    print('Mean Squared Error:', res)
    best_model_name = min(res, key=itemgetter(1))
    return grid[best_model_name]


def test_dataset(grid, X_test, y_test):
    res = {}
    for name, model in grid.items():
        y_pred = model.predict(X_test)
        res[name] = {'MSE': mean_squared_error(y_test, y_pred),
                       'R2': r2_score(y_test, y_pred)
                      }
    return res

def predict(grid, X_test, model_name):
    return grid[model_name].predict(X_test)


def print_grid_results(grids):
    for name, model in grids.items():
        print('{:-^70}'.format(' [' + name + '] '))
        print('Score:\t\t{:.2%}'.format(model.best_score_))
        print('Parameters:\t{}'.format(model.best_params_))
        print('*' * 70)


models = [
    {   'name':     'SVR_rbf',
        'model':    SVR(),
        'title':    "SVR_rbf",
        'param_grid': {
            'SVR_rbf__C':           [0.1, 1, 5, 10],
            'SVR_rbf__max_iter':    [500]
        } 
    },
    {   'name':     'SVR_linear',
        'model':      SVR(kernel='linear'),
        'title':    "SVR_rbf",
        'param_grid': {
            'SVR_linear__C':           [0.01, 0.1, 1, 5],
            'SVR_linear__max_iter':    [1000]
        } 
    },
    {   'name':     'Ridge',
        'model':    Ridge(),
        'title':    "Ridge",
        'param_grid': {
            'Ridge__alpha':         [0.1, 0.5, 5, 10, 50, 100, 500],
            'Ridge__max_iter':      [200]
        } 
    },
    {   'name':     'Lasso',
        'model':    Lasso(),
        'title':    "Lasso",
        'param_grid':  {
            'Lasso__alpha':         [0.0001, 0.001, 0.01, 0.1, 1, 10],
            'Lasso__max_iter':      [1000]
        } 
    },
    {   'name':     'RandomForest',
        'model':    RandomForestRegressor(),
        'title':    "RandomForest",
        'param_grid':  {
            'RandomForest__n_estimators':   [50, 250, 500],
            'RandomForest__max_depth':      [5],
        } 
    },
]


def main(path):
    os.chdir(str(path))

    X_train, X_test, y_train, y_test = \
        get_data_split(path, test_size=127/510.)
    grid = tune_models_hyperparams(X_train, y_train, models, cv=3,
                                   verbose=2, n_jobs=-1)
    print_grid_results(grid)
    model = get_best_model(grid, X_test, y_test)
    df = pd.DataFrame({'predicted': model.predict(X_test)})
    df.to_csv('predicted.csv', index=False)

if __name__ == "__main__":
    p =  Path(__file__).parent.resolve()
    main(p)

Я согласен с @George - "there is something "wrong" with the test set". Я получил аналогичные результаты MSE - прим. 21.

Я также попытался объединить наборы данных для обучения и тестирования и передать их в GridSearchCV.

Вот результаты этих попыток:

In [33]: print_grid_results(grid)
----------------------------- [SVR_rbf] ------------------------------
Score:          48.98%
Parameters:     {'SVR_rbf__C': 5, 'SVR_rbf__max_iter': 500}
**********************************************************************
---------------------------- [SVR_linear] ----------------------------
Score:          64.07%
Parameters:     {'SVR_linear__C': 0.1, 'SVR_linear__max_iter': 500}
**********************************************************************
------------------------------ [Ridge] -------------------------------
Score:          63.98%
Parameters:     {'Ridge__alpha': 100, 'Ridge__max_iter': 200}
**********************************************************************
------------------------------ [Lasso] -------------------------------
Score:          60.36%
Parameters:     {'Lasso__alpha': 0.001, 'Lasso__max_iter': 1000}
**********************************************************************
--------------------------- [RandomForest] ---------------------------
Score:          44.01%
Parameters:     {'RandomForest__max_depth': 5, 'RandomForest__n_estimators': 100}
**********************************************************************

Также разные сплиты дают очень разные результаты тестов:

In [43]: clf = grid['SVR_linear']

In [44]: {k:v for k,v in clf.cv_results_.items() if k.endswith('_test_score')}
Out[44]:
{'mean_test_score': array([0.64067998, 0.63919104, 0.6391681 , 0.64067998, 0.63919104, 0.6391681 , 0.64067998, 0.63919104, 0.6391681 ]),
 'rank_test_score': array([1, 4, 7, 1, 4, 7, 1, 4, 7]),
 'split0_test_score': array([0.98557453, 0.98876705, 0.98883802, 0.98557453, 0.98876705, 0.98883802, 0.98557453, 0.98876705, 0.98883802]),
 'split1_test_score': array([0.69915178, 0.69750946, 0.69740475, 0.69915178, 0.69750946, 0.69740475, 0.69915178, 0.69750946, 0.69740475]),
 'split2_test_score': array([0.23568677, 0.22964765, 0.22961214, 0.23568677, 0.22964765, 0.22961214, 0.23568677, 0.22964765, 0.22961214]),
 'std_test_score': array([0.30903146, 0.31275403, 0.31278954, 0.30903146, 0.31275403, 0.31278954, 0.30903146, 0.31275403, 0.31278954])}

Вот полный код:

import os
#import contextlib
from pathlib import Path
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import SGDRegressor, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVR
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.externals import joblib


def get_data_split(path='.'):
    p = Path(path)
    kwargs = dict(delim_whitespace=True, header=None)
    X_train = pd.read_csv(list(p.glob('trainX.txt*'))[0], **kwargs)
    y_train = pd.read_csv(list(p.glob('trainY.txt*'))[0], **kwargs)
    X_test = pd.read_csv(list(p.glob('testX.txt*'))[0], **kwargs)
    y_test = pd.read_csv(list(p.glob('testY.txt*'))[0], **kwargs)
    return X_train, y_train[0], X_test, y_test[0]


def get_data(path='.'):
    p = Path(path)
    kwargs = dict(delim_whitespace=True, header=None)
    X_train = pd.read_csv(list(p.glob('trainX.txt*'))[0], **kwargs)
    y_train = pd.read_csv(list(p.glob('trainY.txt*'))[0], **kwargs)
    X_test = pd.read_csv(list(p.glob('testX.txt*'))[0], **kwargs)
    y_test = pd.read_csv(list(p.glob('testY.txt*'))[0], **kwargs)
    return (pd.concat([X_train, X_test], ignore_index=True),
            pd.concat([y_train, y_test], ignore_index=True)[0])


def fit_all_classifiers_grid(X, y, classifiers, **common_grid_kwargs):
    grids = {}
    for clf in classifiers:
        print('{:-^70}'.format(' [' + clf['name'] + '] '))
        pipe = Pipeline([
                    ("scale", StandardScaler()),
                    (clf['name'], clf['clf'])   ])
        grids[clf['name']] = (GridSearchCV(pipe,
                                           param_grid=clf['param_grid'],
                                           **common_grid_kwargs)
                                  .fit(X, y))
        # saving single trained model ...
        joblib.dump(grids[clf['name']], './{}.pkl'.format(clf['name']))
    return grids


def test_dataset(grid, X_test, y_test):
    res = {}
    for name, clf in grid.items():
        y_pred = clf.predict(X_test)
        res[name] = {'MSE': mean_squared_error(y_test, y_pred),
                       'R2': r2_score(y_test, y_pred)
                      }
    return res


def print_grid_results(grids):
    for name, clf in grids.items():
        print('{:-^70}'.format(' [' + name + '] '))
        print('Score:\t\t{:.2%}'.format(clf.best_score_))
        print('Parameters:\t{}'.format(clf.best_params_))
        print('*' * 70)




classifiers = [
    {   'name':     'SVR_rbf',
        'clf':      SVR(),
        'title':    "SVR_rbf",
        'param_grid': {
            'SVR_rbf__C':           [0.1, 1, 5],
            'SVR_rbf__max_iter':    [500, 1000, 5000]
        } 
    },
    {   'name':     'SVR_linear',
        'clf':      SVR(kernel='linear'),
        'title':    "SVR_rbf",
        'param_grid': {
            'SVR_linear__C':           [0.1, 1, 5],
            'SVR_linear__max_iter':    [500, 1000, 5000]
        } 
    },
    {   'name':     'Ridge',
        'clf':      Ridge(),
        'title':    "Ridge",
        'param_grid': {
            'Ridge__alpha':         [0.1, 1, 5, 10, 50, 100],
            'Ridge__max_iter':      [200, 500]
        } 
    },
    {   'name':     'Lasso',
        'clf':      Lasso(),
        'title':    "Lasso",
        'param_grid':  {
            'Lasso__alpha':         [0.001, 0.01, 0.1, 1, 5, 10],
            'Lasso__max_iter':      [1000, 5000]
        } 
    },
    {   'name':     'RandomForest',
        'clf':      RandomForestRegressor(),
        'title':    "RandomForest",
        'param_grid':  {
            'RandomForest__n_estimators':   [10, 100],
            'RandomForest__max_depth':      [3, 5],
        } 
    },
]


def main(path):
    #path = r'D:\data\work\.ML\SO\49094242-SVM provided a bad result in my data'
    os.chdir(path)

    X, y = get_data(path)
    grid = fit_all_classifiers_grid(X, y, classifiers, cv=3, verbose=2, n_jobs=-1)
    print_grid_results(grid)

    #X_train, y_train, X_test, y_test = get_data_split(path)
    #grid = fit_all_classifiers_grid(X_train, y_train, classifiers, cv=2, verbose=2, n_jobs=-1)
    #res = test_dataset(grid, X_test, y_test)
    #print(res)

PS извините за использование имени classifier вместо regressor - я просто повторно использовал свой старый код, где искал лучший классификатор....

согласно вашему набору данных кажется, что ваши характеристики слишком высоки. Лучше использовать алгоритмы группировки признаков до того, как вы начнете обработку с помощью SVM.