multi_task_learning/train_alternating.m at master · argyriou/multi_task_learning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
function [W,D,costfunc] = train_alternating(trainx,trainy,task_indexes,gamma,Dini,iterations,...
    method,kernel_method,f_method,Dmin_method)

% Main algorithm for Multi-task Feature Learning (with a linear kernel)
% See [Argyriou,Evgeniou,Pontil, NIPS 2006, ML journal 2007]
%
% task_indexes : sizes of samples per task (may be unbalanced)
% gamma        : regularization parameter
% method :  feat        = orthonormal feature learning, i.e. using trace(W' f(D) W)
%           independent = learning with no coupling across tasks (i.e.
%                         using ||W||_2 regularization)
%           diagonal    = variable (feature) selection (i.e. D is diagonal)
% kernel_method : method for kernel learning (e.g. SVM, least square
%                 regression etc.)
% f_method    : evaluates f(D) (acts on the singular values of D)
% Dmin_method : method for minimizing over D of the form
%               min_d { sum_i f(d_i) b_i^2 }
%               (b_i are the singular values of W,
%               or in case of var. selection the L2 norms of the rows of W)

num_data = size(trainx,2);
dim = size(trainx,1);
T = length(task_indexes);

feat = 1; independent = 2; diagonal = 3;

if (max(max(abs(Dini-Dini'))) > eps)
    error('D should be symmetric');
end
if (min(eig(Dini)) < -eps)
    error('D should be positive semidefinite');
end
if (abs(trace(Dini)-1) > 100*eps)
    error('D should have trace  1');
end
D = Dini;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                                            %
%               Feature Learning                             %
%                                                            %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

if (method == feat)
    costfunc = [];

    % Compute f(D)^{-1/2} for the next step
    [U,S,dummy] = svd(D);               % svd seems more robust than eig
    fS = feval(f_method,diag(S));
    temp = sqrt(fS);
    tempi = find(temp > eps);
    temp(tempi) = 1./temp(tempi);
    fD_isqrt = U * diag(temp) * U';

    for iter = 1:iterations
        % Use variable transform to solve the regularization problem for
        % fixed D
        new_trainx = fD_isqrt * trainx;
        [W,costf,err,reg] = train_kernel(new_trainx,trainy,task_indexes,gamma,kernel_method);
        W = fD_isqrt * W;

        costfunc = [costfunc; iter, costf, err, reg];

        % Update D
        [U,S,V] = svd(W);
        if (dim > T)
            S = [S, zeros(dim,dim-T)];
        end
        Smin = feval(Dmin_method, diag(S));
        D = U * diag(Smin) * U';

        % Compute f(D)^{-1/2} for the next step
        fS = feval(f_method,Smin);
        temp = sqrt(fS);
        tempi = find(temp > eps);
        temp(tempi) = 1./temp(tempi);
        fD_isqrt = U * diag(temp) * U';
    end
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                                          %
%           Independent Regularizations                    %
%                                                          %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

if (method == independent)
    [W,costfunc,err,reg] = train_kernel(trainx,trainy,task_indexes,gamma,kernel_method);
    D = [];
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                                          %
%           Variable selection                             %
%                                                          %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

if (method == diagonal)
    if (norm(D-diag(diag(D))) > eps)
        error('D should be diagonal');
    end
    costfunc = [];

    % Compute f(D)^{-1/2} for the next step
    fS = feval(f_method,diag(D));
    temp = sqrt(fS);
    tempi = find(temp > eps);
    temp(tempi) = 1./temp(tempi);
    fD_isqrt = diag(temp);

    for iter = 1:iterations
        new_trainx = fD_isqrt * trainx;
        [W,costf,err,reg] = train_kernel(new_trainx,trainy,task_indexes,gamma,kernel_method);
        W = fD_isqrt * W;

        costfunc = [costfunc; iter, costf, err, reg];

        % Update D
        Smin = feval(Dmin_method, sqrt(sum(W.^2,2)));
        D = diag(Smin);

        % Compute f(D)^{-1/2} for the next step
        fS = feval(f_method,Smin);
        temp = sqrt(fS);
        tempi = find(temp > eps);
        temp(tempi) = 1./temp(tempi);
        fD_isqrt = diag(temp);
    end
end