feat: add Camera Work AI package

CameraAI~/Tools~/CWCameraWorker/_internal/sklearn/linear_model/_cd_fast.pyx

@@ -0,0 +1,962 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from libc.math cimport fabs
+import numpy as np
+
+from cython cimport floating
+import warnings
+from ..exceptions import ConvergenceWarning
+
+from ..utils._cython_blas cimport (
+    _axpy, _dot, _asum, _gemv, _nrm2, _copy, _scal
+)
+from ..utils._cython_blas cimport ColMajor, Trans, NoTrans
+from ..utils._typedefs cimport uint32_t
+from ..utils._random cimport our_rand_r
+
+
+# The following two functions are shamelessly copied from the tree code.
+
+cdef enum:
+    # Max value for our rand_r replacement (near the bottom).
+    # We don't use RAND_MAX because it's different across platforms and
+    # particularly tiny on Windows/MSVC.
+    # It corresponds to the maximum representable value for
+    # 32-bit signed integers (i.e. 2^31 - 1).
+    RAND_R_MAX = 2147483647
+
+
+cdef inline uint32_t rand_int(uint32_t end, uint32_t* random_state) noexcept nogil:
+    """Generate a random integer in [0; end)."""
+    return our_rand_r(random_state) % end
+
+
+cdef inline floating fmax(floating x, floating y) noexcept nogil:
+    if x > y:
+        return x
+    return y
+
+
+cdef inline floating fsign(floating f) noexcept nogil:
+    if f == 0:
+        return 0
+    elif f > 0:
+        return 1.0
+    else:
+        return -1.0
+
+
+cdef floating abs_max(int n, const floating* a) noexcept nogil:
+    """np.max(np.abs(a))"""
+    cdef int i
+    cdef floating m = fabs(a[0])
+    cdef floating d
+    for i in range(1, n):
+        d = fabs(a[i])
+        if d > m:
+            m = d
+    return m
+
+
+cdef floating max(int n, floating* a) noexcept nogil:
+    """np.max(a)"""
+    cdef int i
+    cdef floating m = a[0]
+    cdef floating d
+    for i in range(1, n):
+        d = a[i]
+        if d > m:
+            m = d
+    return m
+
+
+cdef floating diff_abs_max(int n, const floating* a, floating* b) noexcept nogil:
+    """np.max(np.abs(a - b))"""
+    cdef int i
+    cdef floating m = fabs(a[0] - b[0])
+    cdef floating d
+    for i in range(1, n):
+        d = fabs(a[i] - b[i])
+        if d > m:
+            m = d
+    return m
+
+
+message_conv = (
+    "Objective did not converge. You might want to increase "
+    "the number of iterations, check the scale of the "
+    "features or consider increasing regularisation."
+)
+
+
+message_ridge = (
+    "Linear regression models with a zero l1 penalization "
+    "strength are more efficiently fitted using one of the "
+    "solvers implemented in "
+    "sklearn.linear_model.Ridge/RidgeCV instead."
+)
+
+
+def enet_coordinate_descent(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[::1, :] X,
+    const floating[::1] y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0
+):
+    """Cython version of the coordinate descent algorithm
+        for Elastic-Net regression
+
+        We minimize
+
+        (1/2) * norm(y - X w, 2)^2 + alpha norm(w, 1) + (beta/2) norm(w, 2)^2
+
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
+    """
+
+    if floating is float:
+        dtype = np.float32
+    else:
+        dtype = np.float64
+
+    # get the data information into easy vars
+    cdef unsigned int n_samples = X.shape[0]
+    cdef unsigned int n_features = X.shape[1]
+
+    # compute norms of the columns of X
+    cdef floating[::1] norm_cols_X = np.square(X).sum(axis=0)
+
+    # initial value of the residuals
+    cdef floating[::1] R = np.empty(n_samples, dtype=dtype)
+    cdef floating[::1] XtA = np.empty(n_features, dtype=dtype)
+
+    cdef floating tmp
+    cdef floating w_ii
+    cdef floating d_w_max
+    cdef floating w_max
+    cdef floating d_w_ii
+    cdef floating gap = tol + 1.0
+    cdef floating d_w_tol = tol
+    cdef floating dual_norm_XtA
+    cdef floating R_norm2
+    cdef floating w_norm2
+    cdef floating l1_norm
+    cdef floating const_
+    cdef floating A_norm2
+    cdef unsigned int ii
+    cdef unsigned int n_iter = 0
+    cdef unsigned int f_iter
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
+
+    if alpha == 0 and beta == 0:
+        warnings.warn("Coordinate descent with no regularization may lead to "
+                      "unexpected results and is discouraged.")
+
+    with nogil:
+        # R = y - np.dot(X, w)
+        _copy(n_samples, &y[0], 1, &R[0], 1)
+        _gemv(ColMajor, NoTrans, n_samples, n_features, -1.0, &X[0, 0],
+              n_samples, &w[0], 1, 1.0, &R[0], 1)
+
+        # tol *= np.dot(y, y)
+        tol *= _dot(n_samples, &y[0], 1, &y[0], 1)
+
+        for n_iter in range(max_iter):
+            w_max = 0.0
+            d_w_max = 0.0
+            for f_iter in range(n_features):  # Loop over coordinates
+                if random:
+                    ii = rand_int(n_features, rand_r_state)
+                else:
+                    ii = f_iter
+
+                if norm_cols_X[ii] == 0.0:
+                    continue
+
+                w_ii = w[ii]  # Store previous value
+
+                if w_ii != 0.0:
+                    # R += w_ii * X[:,ii]
+                    _axpy(n_samples, w_ii, &X[0, ii], 1, &R[0], 1)
+
+                # tmp = (X[:,ii]*R).sum()
+                tmp = _dot(n_samples, &X[0, ii], 1, &R[0], 1)
+
+                if positive and tmp < 0:
+                    w[ii] = 0.0
+                else:
+                    w[ii] = (fsign(tmp) * fmax(fabs(tmp) - alpha, 0)
+                             / (norm_cols_X[ii] + beta))
+
+                if w[ii] != 0.0:
+                    # R -=  w[ii] * X[:,ii] # Update residual
+                    _axpy(n_samples, -w[ii], &X[0, ii], 1, &R[0], 1)
+
+                # update the maximum absolute coefficient update
+                d_w_ii = fabs(w[ii] - w_ii)
+                d_w_max = fmax(d_w_max, d_w_ii)
+
+                w_max = fmax(w_max, fabs(w[ii]))
+
+            if (
+                w_max == 0.0
+                or d_w_max / w_max < d_w_tol
+                or n_iter == max_iter - 1
+            ):
+                # the biggest coordinate update of this iteration was smaller
+                # than the tolerance: check the duality gap as ultimate
+                # stopping criterion
+
+                # XtA = np.dot(X.T, R) - beta * w
+                _copy(n_features, &w[0], 1, &XtA[0], 1)
+                _gemv(ColMajor, Trans,
+                      n_samples, n_features, 1.0, &X[0, 0], n_samples,
+                      &R[0], 1,
+                      -beta, &XtA[0], 1)
+
+                if positive:
+                    dual_norm_XtA = max(n_features, &XtA[0])
+                else:
+                    dual_norm_XtA = abs_max(n_features, &XtA[0])
+
+                # R_norm2 = np.dot(R, R)
+                R_norm2 = _dot(n_samples, &R[0], 1, &R[0], 1)
+
+                # w_norm2 = np.dot(w, w)
+                w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
+
+                if (dual_norm_XtA > alpha):
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * (const_ ** 2)
+                    gap = 0.5 * (R_norm2 + A_norm2)
+                else:
+                    const_ = 1.0
+                    gap = R_norm2
+
+                l1_norm = _asum(n_features, &w[0], 1)
+
+                gap += (alpha * l1_norm
+                        - const_ * _dot(n_samples, &R[0], 1, &y[0], 1)  # np.dot(R.T, y)
+                        + 0.5 * beta * (1 + const_ ** 2) * (w_norm2))
+
+                if gap < tol:
+                    # return if we reached desired tolerance
+                    break
+
+        else:
+            # for/else, runs if for doesn't end with a `break`
+            with gil:
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                if alpha < np.finfo(np.float64).eps:
+                    message += "\n" + message_ridge
+                warnings.warn(message, ConvergenceWarning)
+
+    return np.asarray(w), gap, tol, n_iter + 1
+
+
+def sparse_enet_coordinate_descent(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[::1] X_data,
+    const int[::1] X_indices,
+    const int[::1] X_indptr,
+    const floating[::1] y,
+    const floating[::1] sample_weight,
+    const floating[::1] X_mean,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0,
+):
+    """Cython version of the coordinate descent algorithm for Elastic-Net
+
+    We minimize:
+
+        1/2 * norm(y - Z w, 2)^2 + alpha * norm(w, 1) + (beta/2) * norm(w, 2)^2
+
+    where Z = X - X_mean.
+    With sample weights sw, this becomes
+
+        1/2 * sum(sw * (y - Z w)^2, axis=0) + alpha * norm(w, 1)
+        + (beta/2) * norm(w, 2)^2
+
+    and X_mean is the weighted average of X (per column).
+
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
+    """
+    # Notes for sample_weight:
+    # For dense X, one centers X and y and then rescales them by sqrt(sample_weight).
+    # Here, for sparse X, we get the sample_weight averaged center X_mean. We take care
+    # that every calculation results as if we had rescaled y and X (and therefore also
+    # X_mean) by sqrt(sample_weight) without actually calculating the square root.
+    # We work with:
+    #     yw = sample_weight * y
+    #     R = sample_weight * residual
+    #     norm_cols_X = np.sum(sample_weight * (X - X_mean)**2, axis=0)
+
+    if floating is float:
+        dtype = np.float32
+    else:
+        dtype = np.float64
+
+    # get the data information into easy vars
+    cdef unsigned int n_samples = y.shape[0]
+    cdef unsigned int n_features = w.shape[0]
+
+    # compute norms of the columns of X
+    cdef floating[:] norm_cols_X = np.zeros(n_features, dtype=dtype)
+
+    # initial value of the residuals
+    # R = y - Zw, weighted version R = sample_weight * (y - Zw)
+    cdef floating[::1] R
+    cdef floating[::1] XtA = np.empty(n_features, dtype=dtype)
+    cdef const floating[::1] yw
+
+    cdef floating tmp
+    cdef floating w_ii
+    cdef floating d_w_max
+    cdef floating w_max
+    cdef floating d_w_ii
+    cdef floating gap = tol + 1.0
+    cdef floating d_w_tol = tol
+    cdef floating dual_norm_XtA
+    cdef floating X_mean_ii
+    cdef floating R_sum = 0.0
+    cdef floating R_norm2
+    cdef floating w_norm2
+    cdef floating l1_norm
+    cdef floating const_
+    cdef floating A_norm2
+    cdef floating normalize_sum
+    cdef unsigned int ii
+    cdef unsigned int jj
+    cdef unsigned int n_iter = 0
+    cdef unsigned int f_iter
+    cdef unsigned int startptr = X_indptr[0]
+    cdef unsigned int endptr
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
+    cdef bint center = False
+    cdef bint no_sample_weights = sample_weight is None
+    cdef int kk
+
+    if no_sample_weights:
+        yw = y
+        R = y.copy()
+    else:
+        yw = np.multiply(sample_weight, y)
+        R = yw.copy()
+
+    with nogil:
+        # center = (X_mean != 0).any()
+        for ii in range(n_features):
+            if X_mean[ii]:
+                center = True
+                break
+
+        # R = y - np.dot(X, w)
+        for ii in range(n_features):
+            X_mean_ii = X_mean[ii]
+            endptr = X_indptr[ii + 1]
+            normalize_sum = 0.0
+            w_ii = w[ii]
+
+            if no_sample_weights:
+                for jj in range(startptr, endptr):
+                    normalize_sum += (X_data[jj] - X_mean_ii) ** 2
+                    R[X_indices[jj]] -= X_data[jj] * w_ii
+                norm_cols_X[ii] = normalize_sum + \
+                    (n_samples - endptr + startptr) * X_mean_ii ** 2
+                if center:
+                    for jj in range(n_samples):
+                        R[jj] += X_mean_ii * w_ii
+                        R_sum += R[jj]
+            else:
+                # R = sw * (y - np.dot(X, w))
+                for jj in range(startptr, endptr):
+                    tmp = sample_weight[X_indices[jj]]
+                    # second term will be subtracted by loop over range(n_samples)
+                    normalize_sum += (tmp * (X_data[jj] - X_mean_ii) ** 2
+                                      - tmp * X_mean_ii ** 2)
+                    R[X_indices[jj]] -= tmp * X_data[jj] * w_ii
+                if center:
+                    for jj in range(n_samples):
+                        normalize_sum += sample_weight[jj] * X_mean_ii ** 2
+                        R[jj] += sample_weight[jj] * X_mean_ii * w_ii
+                        R_sum += R[jj]
+                norm_cols_X[ii] = normalize_sum
+            startptr = endptr
+
+        # Note: No need to update R_sum from here on because the update terms cancel
+        # each other: w_ii * np.sum(X[:,ii] - X_mean[ii]) = 0. R_sum is only ever
+        # needed and calculated if X_mean is provided.
+
+        # tol *= np.dot(y, y)
+        # with sample weights: tol *= y @ (sw * y)
+        tol *= _dot(n_samples, &y[0], 1, &yw[0], 1)
+
+        for n_iter in range(max_iter):
+
+            w_max = 0.0
+            d_w_max = 0.0
+
+            for f_iter in range(n_features):  # Loop over coordinates
+                if random:
+                    ii = rand_int(n_features, rand_r_state)
+                else:
+                    ii = f_iter
+
+                if norm_cols_X[ii] == 0.0:
+                    continue
+
+                startptr = X_indptr[ii]
+                endptr = X_indptr[ii + 1]
+                w_ii = w[ii]  # Store previous value
+                X_mean_ii = X_mean[ii]
+
+                if w_ii != 0.0:
+                    # R += w_ii * X[:,ii]
+                    if no_sample_weights:
+                        for jj in range(startptr, endptr):
+                            R[X_indices[jj]] += X_data[jj] * w_ii
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] -= X_mean_ii * w_ii
+                    else:
+                        for jj in range(startptr, endptr):
+                            tmp = sample_weight[X_indices[jj]]
+                            R[X_indices[jj]] += tmp * X_data[jj] * w_ii
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] -= sample_weight[jj] * X_mean_ii * w_ii
+
+                # tmp = (X[:,ii] * R).sum()
+                tmp = 0.0
+                for jj in range(startptr, endptr):
+                    tmp += R[X_indices[jj]] * X_data[jj]
+
+                if center:
+                    tmp -= R_sum * X_mean_ii
+
+                if positive and tmp < 0.0:
+                    w[ii] = 0.0
+                else:
+                    w[ii] = fsign(tmp) * fmax(fabs(tmp) - alpha, 0) \
+                            / (norm_cols_X[ii] + beta)
+
+                if w[ii] != 0.0:
+                    # R -=  w[ii] * X[:,ii] # Update residual
+                    if no_sample_weights:
+                        for jj in range(startptr, endptr):
+                            R[X_indices[jj]] -= X_data[jj] * w[ii]
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] += X_mean_ii * w[ii]
+                    else:
+                        for jj in range(startptr, endptr):
+                            tmp = sample_weight[X_indices[jj]]
+                            R[X_indices[jj]] -= tmp * X_data[jj] * w[ii]
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] += sample_weight[jj] * X_mean_ii * w[ii]
+
+                # update the maximum absolute coefficient update
+                d_w_ii = fabs(w[ii] - w_ii)
+                d_w_max = fmax(d_w_max, d_w_ii)
+
+                w_max = fmax(w_max, fabs(w[ii]))
+
+            if w_max == 0.0 or d_w_max / w_max < d_w_tol or n_iter == max_iter - 1:
+                # the biggest coordinate update of this iteration was smaller than
+                # the tolerance: check the duality gap as ultimate stopping
+                # criterion
+
+                # XtA = X.T @ R - beta * w
+                # sparse X.T / dense R dot product
+                for ii in range(n_features):
+                    XtA[ii] = 0.0
+                    for kk in range(X_indptr[ii], X_indptr[ii + 1]):
+                        XtA[ii] += X_data[kk] * R[X_indices[kk]]
+
+                    if center:
+                        XtA[ii] -= X_mean[ii] * R_sum
+                    XtA[ii] -= beta * w[ii]
+
+                if positive:
+                    dual_norm_XtA = max(n_features, &XtA[0])
+                else:
+                    dual_norm_XtA = abs_max(n_features, &XtA[0])
+
+                # R_norm2 = np.dot(R, R)
+                if no_sample_weights:
+                    R_norm2 = _dot(n_samples, &R[0], 1, &R[0], 1)
+                else:
+                    R_norm2 = 0.0
+                    for jj in range(n_samples):
+                        # R is already multiplied by sample_weight
+                        if sample_weight[jj] != 0:
+                            R_norm2 += (R[jj] ** 2) / sample_weight[jj]
+
+                # w_norm2 = np.dot(w, w)
+                w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
+                if (dual_norm_XtA > alpha):
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * const_**2
+                    gap = 0.5 * (R_norm2 + A_norm2)
+                else:
+                    const_ = 1.0
+                    gap = R_norm2
+
+                l1_norm = _asum(n_features, &w[0], 1)
+
+                gap += (alpha * l1_norm
+                        - const_ * _dot(n_samples, &R[0], 1, &y[0], 1)  # np.dot(R.T, y)
+                        + 0.5 * beta * (1 + const_ ** 2) * w_norm2)
+
+                if gap < tol:
+                    # return if we reached desired tolerance
+                    break
+
+        else:
+            # for/else, runs if for doesn't end with a `break`
+            with gil:
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                if alpha < np.finfo(np.float64).eps:
+                    message += "\n" + message_ridge
+                warnings.warn(message, ConvergenceWarning)
+
+    return np.asarray(w), gap, tol, n_iter + 1
+
+
+def enet_coordinate_descent_gram(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[:, ::1] Q,
+    const floating[::1] q,
+    const floating[:] y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0
+):
+    """Cython version of the coordinate descent algorithm
+        for Elastic-Net regression
+
+        We minimize
+
+        (1/2) * w^T Q w - q^T w + alpha norm(w, 1) + (beta/2) * norm(w, 2)^2
+
+        which amount to the Elastic-Net problem when:
+        Q = X^T X (Gram matrix)
+        q = X^T y
+
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
+    """
+
+    if floating is float:
+        dtype = np.float32
+    else:
+        dtype = np.float64
+
+    # get the data information into easy vars
+    cdef unsigned int n_features = Q.shape[0]
+
+    # initial value "Q w" which will be kept of up to date in the iterations
+    cdef floating[:] H = np.dot(Q, w)
+
+    cdef floating[:] XtA = np.zeros(n_features, dtype=dtype)
+    cdef floating tmp
+    cdef floating w_ii
+    cdef floating d_w_max
+    cdef floating w_max
+    cdef floating d_w_ii
+    cdef floating q_dot_w
+    cdef floating w_norm2
+    cdef floating gap = tol + 1.0
+    cdef floating d_w_tol = tol
+    cdef floating dual_norm_XtA
+    cdef unsigned int ii
+    cdef unsigned int n_iter = 0
+    cdef unsigned int f_iter
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
+
+    cdef floating y_norm2 = np.dot(y, y)
+    cdef floating* w_ptr = &w[0]
+    cdef const floating* Q_ptr = &Q[0, 0]
+    cdef const floating* q_ptr = &q[0]
+    cdef floating* H_ptr = &H[0]
+    cdef floating* XtA_ptr = &XtA[0]
+    tol = tol * y_norm2
+
+    if alpha == 0:
+        warnings.warn(
+            "Coordinate descent without L1 regularization may "
+            "lead to unexpected results and is discouraged. "
+            "Set l1_ratio > 0 to add L1 regularization."
+        )
+
+    with nogil:
+        for n_iter in range(max_iter):
+            w_max = 0.0
+            d_w_max = 0.0
+            for f_iter in range(n_features):  # Loop over coordinates
+                if random:
+                    ii = rand_int(n_features, rand_r_state)
+                else:
+                    ii = f_iter
+
+                if Q[ii, ii] == 0.0:
+                    continue
+
+                w_ii = w[ii]  # Store previous value
+
+                if w_ii != 0.0:
+                    # H -= w_ii * Q[ii]
+                    _axpy(n_features, -w_ii, Q_ptr + ii * n_features, 1,
+                          H_ptr, 1)
+
+                tmp = q[ii] - H[ii]
+
+                if positive and tmp < 0:
+                    w[ii] = 0.0
+                else:
+                    w[ii] = fsign(tmp) * fmax(fabs(tmp) - alpha, 0) \
+                        / (Q[ii, ii] + beta)
+
+                if w[ii] != 0.0:
+                    # H +=  w[ii] * Q[ii] # Update H = X.T X w
+                    _axpy(n_features, w[ii], Q_ptr + ii * n_features, 1,
+                          H_ptr, 1)
+
+                # update the maximum absolute coefficient update
+                d_w_ii = fabs(w[ii] - w_ii)
+                if d_w_ii > d_w_max:
+                    d_w_max = d_w_ii
+
+                if fabs(w[ii]) > w_max:
+                    w_max = fabs(w[ii])
+
+            if w_max == 0.0 or d_w_max / w_max < d_w_tol or n_iter == max_iter - 1:
+                # the biggest coordinate update of this iteration was smaller than
+                # the tolerance: check the duality gap as ultimate stopping
+                # criterion
+
+                # q_dot_w = np.dot(w, q)
+                q_dot_w = _dot(n_features, w_ptr, 1, q_ptr, 1)
+
+                for ii in range(n_features):
+                    XtA[ii] = q[ii] - H[ii] - beta * w[ii]
+                if positive:
+                    dual_norm_XtA = max(n_features, XtA_ptr)
+                else:
+                    dual_norm_XtA = abs_max(n_features, XtA_ptr)
+
+                # temp = np.sum(w * H)
+                tmp = 0.0
+                for ii in range(n_features):
+                    tmp += w[ii] * H[ii]
+                R_norm2 = y_norm2 + tmp - 2.0 * q_dot_w
+
+                # w_norm2 = np.dot(w, w)
+                w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
+
+                if (dual_norm_XtA > alpha):
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * (const_ ** 2)
+                    gap = 0.5 * (R_norm2 + A_norm2)
+                else:
+                    const_ = 1.0
+                    gap = R_norm2
+
+                # The call to asum is equivalent to the L1 norm of w
+                gap += (
+                    alpha * _asum(n_features, &w[0], 1)
+                    - const_ * y_norm2
+                    + const_ * q_dot_w
+                    + 0.5 * beta * (1 + const_ ** 2) * w_norm2
+                )
+
+                if gap < tol:
+                    # return if we reached desired tolerance
+                    break
+
+        else:
+            # for/else, runs if for doesn't end with a `break`
+            with gil:
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                warnings.warn(message, ConvergenceWarning)
+
+    return np.asarray(w), gap, tol, n_iter + 1
+
+
+def enet_coordinate_descent_multi_task(
+    const floating[::1, :] W,
+    floating l1_reg,
+    floating l2_reg,
+    const floating[::1, :] X,
+    const floating[::1, :] Y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0
+):
+    """Cython version of the coordinate descent algorithm
+        for Elastic-Net multi-task regression
+
+        We minimize
+
+        0.5 * norm(Y - X W.T, 2)^2 + l1_reg ||W.T||_21 + 0.5 * l2_reg norm(W.T, 2)^2
+
+    Returns
+    -------
+    W : ndarray of shape (n_tasks, n_features)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
+    """
+
+    if floating is float:
+        dtype = np.float32
+    else:
+        dtype = np.float64
+
+    # get the data information into easy vars
+    cdef unsigned int n_samples = X.shape[0]
+    cdef unsigned int n_features = X.shape[1]
+    cdef unsigned int n_tasks = Y.shape[1]
+
+    # to store XtA
+    cdef floating[:, ::1] XtA = np.zeros((n_features, n_tasks), dtype=dtype)
+    cdef floating XtA_axis1norm
+    cdef floating dual_norm_XtA
+
+    # initial value of the residuals
+    cdef floating[::1, :] R = np.zeros((n_samples, n_tasks), dtype=dtype, order='F')
+
+    cdef floating[::1] norm_cols_X = np.zeros(n_features, dtype=dtype)
+    cdef floating[::1] tmp = np.zeros(n_tasks, dtype=dtype)
+    cdef floating[::1] w_ii = np.zeros(n_tasks, dtype=dtype)
+    cdef floating d_w_max
+    cdef floating w_max
+    cdef floating d_w_ii
+    cdef floating nn
+    cdef floating W_ii_abs_max
+    cdef floating gap = tol + 1.0
+    cdef floating d_w_tol = tol
+    cdef floating R_norm
+    cdef floating w_norm
+    cdef floating ry_sum
+    cdef floating l21_norm
+    cdef unsigned int ii
+    cdef unsigned int jj
+    cdef unsigned int n_iter = 0
+    cdef unsigned int f_iter
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
+
+    cdef const floating* X_ptr = &X[0, 0]
+    cdef const floating* Y_ptr = &Y[0, 0]
+
+    if l1_reg == 0:
+        warnings.warn(
+            "Coordinate descent with l1_reg=0 may lead to unexpected"
+            " results and is discouraged."
+        )
+
+    with nogil:
+        # norm_cols_X = (np.asarray(X) ** 2).sum(axis=0)
+        for ii in range(n_features):
+            norm_cols_X[ii] = _nrm2(n_samples, X_ptr + ii * n_samples, 1) ** 2
+
+        # R = Y - np.dot(X, W.T)
+        _copy(n_samples * n_tasks, Y_ptr, 1, &R[0, 0], 1)
+        for ii in range(n_features):
+            for jj in range(n_tasks):
+                if W[jj, ii] != 0:
+                    _axpy(n_samples, -W[jj, ii], X_ptr + ii * n_samples, 1,
+                          &R[0, jj], 1)
+
+        # tol = tol * linalg.norm(Y, ord='fro') ** 2
+        tol = tol * _nrm2(n_samples * n_tasks, Y_ptr, 1) ** 2
+
+        for n_iter in range(max_iter):
+            w_max = 0.0
+            d_w_max = 0.0
+            for f_iter in range(n_features):  # Loop over coordinates
+                if random:
+                    ii = rand_int(n_features, rand_r_state)
+                else:
+                    ii = f_iter
+
+                if norm_cols_X[ii] == 0.0:
+                    continue
+
+                # w_ii = W[:, ii] # Store previous value
+                _copy(n_tasks, &W[0, ii], 1, &w_ii[0], 1)
+
+                # Using Numpy:
+                # R += np.dot(X[:, ii][:, None], w_ii[None, :]) # rank 1 update
+                # Using Blas Level2:
+                # _ger(RowMajor, n_samples, n_tasks, 1.0,
+                #      &X[0, ii], 1,
+                #      &w_ii[0], 1, &R[0, 0], n_tasks)
+                # Using Blas Level1 and for loop to avoid slower threads
+                # for such small vectors
+                for jj in range(n_tasks):
+                    if w_ii[jj] != 0:
+                        _axpy(n_samples, w_ii[jj], X_ptr + ii * n_samples, 1,
+                              &R[0, jj], 1)
+
+                # Using numpy:
+                # tmp = np.dot(X[:, ii][None, :], R).ravel()
+                # Using BLAS Level 2:
+                # _gemv(RowMajor, Trans, n_samples, n_tasks, 1.0, &R[0, 0],
+                #       n_tasks, &X[0, ii], 1, 0.0, &tmp[0], 1)
+                # Using BLAS Level 1 (faster for small vectors like here):
+                for jj in range(n_tasks):
+                    tmp[jj] = _dot(n_samples, X_ptr + ii * n_samples, 1,
+                                   &R[0, jj], 1)
+
+                # nn = sqrt(np.sum(tmp ** 2))
+                nn = _nrm2(n_tasks, &tmp[0], 1)
+
+                # W[:, ii] = tmp * fmax(1. - l1_reg / nn, 0) / (norm_cols_X[ii] + l2_reg)
+                _copy(n_tasks, &tmp[0], 1, &W[0, ii], 1)
+                _scal(n_tasks, fmax(1. - l1_reg / nn, 0) / (norm_cols_X[ii] + l2_reg),
+                      &W[0, ii], 1)
+
+                # Using numpy:
+                # R -= np.dot(X[:, ii][:, None], W[:, ii][None, :])
+                # Using BLAS Level 2:
+                # Update residual : rank 1 update
+                # _ger(RowMajor, n_samples, n_tasks, -1.0,
+                #      &X[0, ii], 1, &W[0, ii], 1,
+                #      &R[0, 0], n_tasks)
+                # Using BLAS Level 1 (faster for small vectors like here):
+                for jj in range(n_tasks):
+                    if W[jj, ii] != 0:
+                        _axpy(n_samples, -W[jj, ii], X_ptr + ii * n_samples, 1,
+                              &R[0, jj], 1)
+
+                # update the maximum absolute coefficient update
+                d_w_ii = diff_abs_max(n_tasks, &W[0, ii], &w_ii[0])
+
+                if d_w_ii > d_w_max:
+                    d_w_max = d_w_ii
+
+                W_ii_abs_max = abs_max(n_tasks, &W[0, ii])
+                if W_ii_abs_max > w_max:
+                    w_max = W_ii_abs_max
+
+            if w_max == 0.0 or d_w_max / w_max < d_w_tol or n_iter == max_iter - 1:
+                # the biggest coordinate update of this iteration was smaller than
+                # the tolerance: check the duality gap as ultimate stopping
+                # criterion
+
+                # XtA = np.dot(X.T, R) - l2_reg * W.T
+                for ii in range(n_features):
+                    for jj in range(n_tasks):
+                        XtA[ii, jj] = _dot(
+                            n_samples, X_ptr + ii * n_samples, 1, &R[0, jj], 1
+                            ) - l2_reg * W[jj, ii]
+
+                # dual_norm_XtA = np.max(np.sqrt(np.sum(XtA ** 2, axis=1)))
+                dual_norm_XtA = 0.0
+                for ii in range(n_features):
+                    # np.sqrt(np.sum(XtA ** 2, axis=1))
+                    XtA_axis1norm = _nrm2(n_tasks, &XtA[ii, 0], 1)
+                    if XtA_axis1norm > dual_norm_XtA:
+                        dual_norm_XtA = XtA_axis1norm
+
+                # TODO: use squared L2 norm directly
+                # R_norm = linalg.norm(R, ord='fro')
+                # w_norm = linalg.norm(W, ord='fro')
+                R_norm = _nrm2(n_samples * n_tasks, &R[0, 0], 1)
+                w_norm = _nrm2(n_features * n_tasks, &W[0, 0], 1)
+                if (dual_norm_XtA > l1_reg):
+                    const_ = l1_reg / dual_norm_XtA
+                    A_norm = R_norm * const_
+                    gap = 0.5 * (R_norm ** 2 + A_norm ** 2)
+                else:
+                    const_ = 1.0
+                    gap = R_norm ** 2
+
+                # ry_sum = np.sum(R * y)
+                ry_sum = _dot(n_samples * n_tasks, &R[0, 0], 1, &Y[0, 0], 1)
+
+                # l21_norm = np.sqrt(np.sum(W ** 2, axis=0)).sum()
+                l21_norm = 0.0
+                for ii in range(n_features):
+                    l21_norm += _nrm2(n_tasks, &W[0, ii], 1)
+
+                gap += (
+                    l1_reg * l21_norm
+                    - const_ * ry_sum
+                    + 0.5 * l2_reg * (1 + const_ ** 2) * (w_norm ** 2)
+                )
+
+                if gap <= tol:
+                    # return if we reached desired tolerance
+                    break
+        else:
+            # for/else, runs if for doesn't end with a `break`
+            with gil:
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                warnings.warn(message, ConvergenceWarning)
+
+    return np.asarray(W), gap, tol, n_iter + 1

CameraAI~/Tools~/CWCameraWorker/_internal/sklearn/linear_model/_sag_fast.pyx.tp

@@ -0,0 +1,642 @@
+{{py:
+
+"""
+
+Template file for easily generate fused types consistent code using Tempita
+(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
+
+Generated file: sag_fast.pyx
+
+Each class is duplicated for all dtypes (float and double). The keywords
+between double braces are substituted during the build.
+"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+# name_suffix, c_type, np_type
+dtypes = [('64', 'double', 'np.float64'),
+          ('32', 'float', 'np.float32')]
+
+}}
+"""SAG and SAGA implementation"""
+
+import numpy as np
+from libc.math cimport exp, fabs, isfinite, log
+from libc.time cimport time, time_t
+from libc.stdio cimport printf
+
+from .._loss._loss cimport (
+    CyLossFunction,
+    CyHalfBinomialLoss,
+    CyHalfMultinomialLoss,
+    CyHalfSquaredError,
+)
+from ..utils._seq_dataset cimport SequentialDataset32, SequentialDataset64
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef inline {{c_type}} fmax{{name_suffix}}({{c_type}} x, {{c_type}} y) noexcept nogil:
+    if x > y:
+        return x
+    return y
+
+{{endfor}}
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef inline {{c_type}} _soft_thresholding{{name_suffix}}({{c_type}} x, {{c_type}} shrinkage) noexcept nogil:
+    return fmax{{name_suffix}}(x - shrinkage, 0) - fmax{{name_suffix}}(- x - shrinkage, 0)
+
+{{endfor}}
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+def sag{{name_suffix}}(
+    SequentialDataset{{name_suffix}} dataset,
+    {{c_type}}[:, ::1] weights_array,
+    {{c_type}}[::1] intercept_array,
+    int n_samples,
+    int n_features,
+    int n_classes,
+    double tol,
+    int max_iter,
+    str loss_function,
+    double step_size,
+    double alpha,
+    double beta,
+    {{c_type}}[:, ::1] sum_gradient_init,
+    {{c_type}}[:, ::1] gradient_memory_init,
+    bint[::1] seen_init,
+    int num_seen,
+    bint fit_intercept,
+    {{c_type}}[::1] intercept_sum_gradient_init,
+    double intercept_decay,
+    bint saga,
+    bint verbose
+):
+    """Stochastic Average Gradient (SAG) and SAGA solvers.
+
+    Used in Ridge and LogisticRegression.
+
+    Some implementation details:
+
+    - Just-in-time (JIT) update: In SAG(A), the average-gradient update is
+    collinear with the drawn sample X_i. Therefore, if the data is sparse, the
+    random sample X_i will change the average gradient only on features j where
+    X_ij != 0. In some cases, the average gradient on feature j might change
+    only after k random samples with no change. In these cases, instead of
+    applying k times the same gradient step on feature j, we apply the gradient
+    step only once, scaled by k. This is called the "just-in-time update", and
+    it is performed in `lagged_update{{name_suffix}}`. This function also
+    applies the proximal operator after the gradient step (if L1 regularization
+    is used in SAGA).
+
+    - Weight scale: In SAG(A), the weights are scaled down at each iteration
+    due to the L2 regularization. To avoid updating all the weights at each
+    iteration, the weight scale is factored out in a separate variable `wscale`
+    which is only used in the JIT update. When this variable is too small, it
+    is reset for numerical stability using the function
+    `scale_weights{{name_suffix}}`. This reset requires applying all remaining
+    JIT updates. This reset is also performed every `n_samples` iterations
+    before each convergence check, so when the algorithm stops, we are sure
+    that there is no remaining JIT updates.
+
+    Reference
+    ---------
+    Schmidt, M., Roux, N. L., & Bach, F. (2013).
+    Minimizing finite sums with the stochastic average gradient
+    https://hal.inria.fr/hal-00860051/document
+    (section 4.3)
+
+    :arxiv:`Defazio, A., Bach F. & Lacoste-Julien S. (2014).
+    "SAGA: A Fast Incremental Gradient Method With Support
+    for Non-Strongly Convex Composite Objectives" <1407.0202>`
+    """
+    # the data pointer for x, the current sample
+    cdef {{c_type}} *x_data_ptr = NULL
+    # the index pointer for the column of the data
+    cdef int *x_ind_ptr = NULL
+    # the number of non-zero features for current sample
+    cdef int xnnz = -1
+    # the label value for current sample
+    # the label value for current sample
+    cdef {{c_type}} y
+    # the sample weight
+    cdef {{c_type}} sample_weight
+
+    # helper variable for indexes
+    cdef int f_idx, s_idx, feature_ind, class_ind, j
+    # the number of pass through all samples
+    cdef int n_iter = 0
+    # helper to track iterations through samples
+    cdef int sample_itr
+    # the index (row number) of the current sample
+    cdef int sample_ind
+
+    # the maximum change in weights, used to compute stopping criteria
+    cdef {{c_type}} max_change
+    # a holder variable for the max weight, used to compute stopping criteria
+    cdef {{c_type}} max_weight
+
+    # the start time of the fit
+    cdef time_t start_time
+    # the end time of the fit
+    cdef time_t end_time
+
+    # precomputation since the step size does not change in this implementation
+    cdef {{c_type}} wscale_update = 1.0 - step_size * alpha
+
+    # helper for cumulative sum
+    cdef {{c_type}} cum_sum
+
+    # the pointer to the coef_ or weights
+    cdef {{c_type}}* weights = &weights_array[0, 0]
+
+    # the sum of gradients for each feature
+    cdef {{c_type}}* sum_gradient = &sum_gradient_init[0, 0]
+
+    # the previously seen gradient for each sample
+    cdef {{c_type}}* gradient_memory = &gradient_memory_init[0, 0]
+
+    # the cumulative sums needed for JIT params
+    cdef {{c_type}}[::1] cumulative_sums = np.empty(n_samples, dtype={{np_type}}, order="c")
+
+    # the index for the last time this feature was updated
+    cdef int[::1] feature_hist = np.zeros(n_features, dtype=np.int32, order="c")
+
+    # the previous weights to use to compute stopping criteria
+    cdef {{c_type}}[:, ::1] previous_weights_array = np.zeros((n_features, n_classes), dtype={{np_type}}, order="c")
+    cdef {{c_type}}* previous_weights = &previous_weights_array[0, 0]
+
+    cdef {{c_type}}[::1] prediction = np.zeros(n_classes, dtype={{np_type}}, order="c")
+
+    cdef {{c_type}}[::1] gradient = np.zeros(n_classes, dtype={{np_type}}, order="c")
+
+    # Intermediate variable that need declaration since cython cannot infer when templating
+    cdef {{c_type}} val
+
+    # Bias correction term in saga
+    cdef {{c_type}} gradient_correction
+
+    # the scalar used for multiplying z
+    cdef {{c_type}} wscale = 1.0
+
+    # return value (-1 if an error occurred, 0 otherwise)
+    cdef int status = 0
+
+    # the cumulative sums for each iteration for the sparse implementation
+    cumulative_sums[0] = 0.0
+
+    # the multipliative scale needed for JIT params
+    cdef {{c_type}}[::1] cumulative_sums_prox
+    cdef {{c_type}}* cumulative_sums_prox_ptr
+
+    cdef bint prox = beta > 0 and saga
+
+    # Loss function to optimize
+    cdef CyLossFunction loss
+    # Whether the loss function is multinomial
+    cdef bint multinomial = False
+    # Multinomial loss function
+    cdef CyHalfMultinomialLoss multiloss
+
+    if loss_function == "multinomial":
+        multinomial = True
+        multiloss = CyHalfMultinomialLoss()
+    elif loss_function == "log":
+        loss = CyHalfBinomialLoss()
+    elif loss_function == "squared":
+        loss = CyHalfSquaredError()
+    else:
+        raise ValueError("Invalid loss parameter: got %s instead of "
+                         "one of ('log', 'squared', 'multinomial')"
+                         % loss_function)
+
+    if prox:
+        cumulative_sums_prox = np.empty(n_samples, dtype={{np_type}}, order="c")
+        cumulative_sums_prox_ptr = &cumulative_sums_prox[0]
+    else:
+        cumulative_sums_prox = None
+        cumulative_sums_prox_ptr = NULL
+
+    with nogil:
+        start_time = time(NULL)
+        for n_iter in range(max_iter):
+            for sample_itr in range(n_samples):
+                # extract a random sample
+                sample_ind = dataset.random(&x_data_ptr, &x_ind_ptr, &xnnz, &y, &sample_weight)
+
+                # cached index for gradient_memory
+                s_idx = sample_ind * n_classes
+
+                # update the number of samples seen and the seen array
+                if seen_init[sample_ind] == 0:
+                    num_seen += 1
+                    seen_init[sample_ind] = 1
+
+                # make the weight updates (just-in-time gradient step, and prox operator)
+                if sample_itr > 0:
+                   status = lagged_update{{name_suffix}}(
+                       weights=weights,
+                       wscale=wscale,
+                       xnnz=xnnz,
+                       n_samples=n_samples,
+                       n_classes=n_classes,
+                       sample_itr=sample_itr,
+                       cumulative_sums=&cumulative_sums[0],
+                       cumulative_sums_prox=cumulative_sums_prox_ptr,
+                       feature_hist=&feature_hist[0],
+                       prox=prox,
+                       sum_gradient=sum_gradient,
+                       x_ind_ptr=x_ind_ptr,
+                       reset=False,
+                       n_iter=n_iter
+                   )
+                   if status == -1:
+                       break
+
+                # find the current prediction
+                predict_sample{{name_suffix}}(
+                    x_data_ptr=x_data_ptr,
+                    x_ind_ptr=x_ind_ptr,
+                    xnnz=xnnz,
+                    w_data_ptr=weights,
+                    wscale=wscale,
+                    intercept=&intercept_array[0],
+                    prediction=&prediction[0],
+                    n_classes=n_classes
+                )
+
+                # compute the gradient for this sample, given the prediction
+                if multinomial:
+                    multiloss.cy_gradient(
+                        y_true=y,
+                        raw_prediction=prediction,
+                        sample_weight=sample_weight,
+                        gradient_out=gradient,
+                    )
+                else:
+                    gradient[0] = loss.cy_gradient(y, prediction[0]) * sample_weight
+
+                # L2 regularization by simply rescaling the weights
+                wscale *= wscale_update
+
+                # make the updates to the sum of gradients
+                for j in range(xnnz):
+                    feature_ind = x_ind_ptr[j]
+                    val = x_data_ptr[j]
+                    f_idx = feature_ind * n_classes
+                    for class_ind in range(n_classes):
+                        gradient_correction = \
+                            val * (gradient[class_ind] -
+                                   gradient_memory[s_idx + class_ind])
+                        if saga:
+                            # Note that this is not the main gradient step,
+                            # which is performed just-in-time in lagged_update.
+                            # This part is done outside the JIT update
+                            # as it does not depend on the average gradient.
+                            # The prox operator is applied after the JIT update
+                            weights[f_idx + class_ind] -= \
+                                (gradient_correction * step_size
+                                 * (1 - 1. / num_seen) / wscale)
+                        sum_gradient[f_idx + class_ind] += gradient_correction
+
+                # fit the intercept
+                if fit_intercept:
+                    for class_ind in range(n_classes):
+                        gradient_correction = (gradient[class_ind] -
+                                               gradient_memory[s_idx + class_ind])
+                        intercept_sum_gradient_init[class_ind] += gradient_correction
+                        gradient_correction *= step_size * (1. - 1. / num_seen)
+                        if saga:
+                            intercept_array[class_ind] -= \
+                                (step_size * intercept_sum_gradient_init[class_ind] /
+                                 num_seen * intercept_decay) + gradient_correction
+                        else:
+                            intercept_array[class_ind] -= \
+                                (step_size * intercept_sum_gradient_init[class_ind] /
+                                 num_seen * intercept_decay)
+
+                        # check to see that the intercept is not inf or NaN
+                        if not isfinite(intercept_array[class_ind]):
+                            status = -1
+                            break
+                    # Break from the n_samples outer loop if an error happened
+                    # in the fit_intercept n_classes inner loop
+                    if status == -1:
+                        break
+
+                # update the gradient memory for this sample
+                for class_ind in range(n_classes):
+                    gradient_memory[s_idx + class_ind] = gradient[class_ind]
+
+                if sample_itr == 0:
+                    cumulative_sums[0] = step_size / (wscale * num_seen)
+                    if prox:
+                        cumulative_sums_prox[0] = step_size * beta / wscale
+                else:
+                    cumulative_sums[sample_itr] = \
+                        (cumulative_sums[sample_itr - 1] +
+                         step_size / (wscale * num_seen))
+                    if prox:
+                        cumulative_sums_prox[sample_itr] = \
+                        (cumulative_sums_prox[sample_itr - 1] +
+                             step_size * beta / wscale)
+                # If wscale gets too small, we need to reset the scale.
+                # This also resets the just-in-time update system.
+                if wscale < 1e-9:
+                    if verbose:
+                        with gil:
+                            print("rescaling...")
+                    status = scale_weights{{name_suffix}}(
+                        weights=weights,
+                        wscale=&wscale,
+                        n_features=n_features,
+                        n_samples=n_samples,
+                        n_classes=n_classes,
+                        sample_itr=sample_itr,
+                        cumulative_sums=&cumulative_sums[0],
+                        cumulative_sums_prox=cumulative_sums_prox_ptr,
+                        feature_hist=&feature_hist[0],
+                        prox=prox,
+                        sum_gradient=sum_gradient,
+                        n_iter=n_iter
+                    )
+                    if status == -1:
+                        break
+
+            # Break from the n_iter outer loop if an error happened in the
+            # n_samples inner loop
+            if status == -1:
+                break
+
+            # We scale the weights every n_samples iterations and reset the
+            # just-in-time update system for numerical stability.
+            # Because this reset is done before every convergence check, we are
+            # sure there is no remaining lagged update when the algorithm stops.
+            status = scale_weights{{name_suffix}}(
+                weights=weights,
+                wscale=&wscale,
+                n_features=n_features,
+                n_samples=n_samples,
+                n_classes=n_classes,
+                sample_itr=n_samples - 1,
+                cumulative_sums=&cumulative_sums[0],
+                cumulative_sums_prox=cumulative_sums_prox_ptr,
+                feature_hist=&feature_hist[0],
+                prox=prox,
+                sum_gradient=sum_gradient,
+                n_iter=n_iter
+            )
+            if status == -1:
+                break
+
+            # check if the stopping criteria is reached
+            max_change = 0.0
+            max_weight = 0.0
+            for idx in range(n_features * n_classes):
+                max_weight = fmax{{name_suffix}}(max_weight, fabs(weights[idx]))
+                max_change = fmax{{name_suffix}}(max_change, fabs(weights[idx] - previous_weights[idx]))
+                previous_weights[idx] = weights[idx]
+            if ((max_weight != 0 and max_change / max_weight <= tol)
+                or max_weight == 0 and max_change == 0):
+                if verbose:
+                    end_time = time(NULL)
+                    with gil:
+                        print("convergence after %d epochs took %d seconds" %
+                              (n_iter + 1, end_time - start_time))
+                break
+            elif verbose:
+                printf('Epoch %d, change: %.8g\n', n_iter + 1,
+                                                  max_change / max_weight)
+    n_iter += 1
+    # We do the error treatment here based on error code in status to avoid
+    # re-acquiring the GIL within the cython code, which slows the computation
+    # when the sag/saga solver is used concurrently in multiple Python threads.
+    if status == -1:
+        raise ValueError(("Floating-point under-/overflow occurred at epoch"
+                          " #%d. Scaling input data with StandardScaler or"
+                          " MinMaxScaler might help.") % n_iter)
+
+    if verbose and n_iter >= max_iter:
+        end_time = time(NULL)
+        print(("max_iter reached after %d seconds") %
+              (end_time - start_time))
+
+    return num_seen, n_iter
+
+{{endfor}}
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef int scale_weights{{name_suffix}}(
+    {{c_type}}* weights,
+    {{c_type}}* wscale,
+    int n_features,
+    int n_samples,
+    int n_classes,
+    int sample_itr,
+    {{c_type}}* cumulative_sums,
+    {{c_type}}* cumulative_sums_prox,
+    int* feature_hist,
+    bint prox,
+    {{c_type}}* sum_gradient,
+    int n_iter
+) noexcept nogil:
+    """Scale the weights and reset wscale to 1.0 for numerical stability, and
+    reset the just-in-time (JIT) update system.
+
+    See `sag{{name_suffix}}`'s docstring about the JIT update system.
+
+    wscale = (1 - step_size * alpha) ** (n_iter * n_samples + sample_itr)
+    can become very small, so we reset it every n_samples iterations to 1.0 for
+    numerical stability. To be able to scale, we first need to update every
+    coefficients and reset the just-in-time update system.
+    This also limits the size of `cumulative_sums`.
+    """
+
+    cdef int status
+    status = lagged_update{{name_suffix}}(
+        weights,
+        wscale[0],
+        n_features,
+        n_samples,
+        n_classes,
+        sample_itr + 1,
+        cumulative_sums,
+        cumulative_sums_prox,
+        feature_hist,
+        prox,
+        sum_gradient,
+        NULL,
+        True,
+        n_iter
+    )
+    # if lagged update succeeded, reset wscale to 1.0
+    if status == 0:
+        wscale[0] = 1.0
+    return status
+
+{{endfor}}
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef int lagged_update{{name_suffix}}(
+    {{c_type}}* weights,
+    {{c_type}} wscale,
+    int xnnz,
+    int n_samples,
+    int n_classes,
+    int sample_itr,
+    {{c_type}}* cumulative_sums,
+    {{c_type}}* cumulative_sums_prox,
+    int* feature_hist,
+    bint prox,
+    {{c_type}}* sum_gradient,
+    int* x_ind_ptr,
+    bint reset,
+    int n_iter
+) noexcept nogil:
+    """Hard perform the JIT updates for non-zero features of present sample.
+
+    See `sag{{name_suffix}}`'s docstring about the JIT update system.
+
+    The updates that awaits are kept in memory using cumulative_sums,
+    cumulative_sums_prox, wscale and feature_hist. See original SAGA paper
+    (Defazio et al. 2014) for details. If reset=True, we also reset wscale to
+    1 (this is done at the end of each epoch).
+    """
+    cdef int feature_ind, class_ind, idx, f_idx, lagged_ind, last_update_ind
+    cdef {{c_type}} cum_sum, grad_step, prox_step, cum_sum_prox
+    for feature_ind in range(xnnz):
+        if not reset:
+            feature_ind = x_ind_ptr[feature_ind]
+        f_idx = feature_ind * n_classes
+
+        cum_sum = cumulative_sums[sample_itr - 1]
+        if prox:
+            cum_sum_prox = cumulative_sums_prox[sample_itr - 1]
+        if feature_hist[feature_ind] != 0:
+            cum_sum -= cumulative_sums[feature_hist[feature_ind] - 1]
+            if prox:
+                cum_sum_prox -= cumulative_sums_prox[feature_hist[feature_ind] - 1]
+        if not prox:
+            for class_ind in range(n_classes):
+                idx = f_idx + class_ind
+                weights[idx] -= cum_sum * sum_gradient[idx]
+                if reset:
+                    weights[idx] *= wscale
+                    if not isfinite(weights[idx]):
+                        # returning here does not require the gil as the return
+                        # type is a C integer
+                        return -1
+        else:
+            for class_ind in range(n_classes):
+                idx = f_idx + class_ind
+                if fabs(sum_gradient[idx] * cum_sum) < cum_sum_prox:
+                    # In this case, we can perform all the gradient steps and
+                    # all the proximal steps in this order, which is more
+                    # efficient than unrolling all the lagged updates.
+                    # Idea taken from scikit-learn-contrib/lightning.
+                    weights[idx] -= cum_sum * sum_gradient[idx]
+                    weights[idx] = _soft_thresholding{{name_suffix}}(weights[idx],
+                                                      cum_sum_prox)
+                else:
+                    last_update_ind = feature_hist[feature_ind]
+                    if last_update_ind == -1:
+                        last_update_ind = sample_itr - 1
+                    for lagged_ind in range(sample_itr - 1,
+                                   last_update_ind - 1, -1):
+                        if lagged_ind > 0:
+                            grad_step = (cumulative_sums[lagged_ind]
+                               - cumulative_sums[lagged_ind - 1])
+                            prox_step = (cumulative_sums_prox[lagged_ind]
+                               - cumulative_sums_prox[lagged_ind - 1])
+                        else:
+                            grad_step = cumulative_sums[lagged_ind]
+                            prox_step = cumulative_sums_prox[lagged_ind]
+                        weights[idx] -= sum_gradient[idx] * grad_step
+                        weights[idx] = _soft_thresholding{{name_suffix}}(weights[idx],
+                                                          prox_step)
+
+                if reset:
+                    weights[idx] *= wscale
+                    # check to see that the weight is not inf or NaN
+                    if not isfinite(weights[idx]):
+                        return -1
+        if reset:
+            feature_hist[feature_ind] = sample_itr % n_samples
+        else:
+            feature_hist[feature_ind] = sample_itr
+
+    if reset:
+        cumulative_sums[sample_itr - 1] = 0.0
+        if prox:
+            cumulative_sums_prox[sample_itr - 1] = 0.0
+
+    return 0
+
+{{endfor}}
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef void predict_sample{{name_suffix}}(
+    {{c_type}}* x_data_ptr,
+    int* x_ind_ptr,
+    int xnnz,
+    {{c_type}}* w_data_ptr,
+    {{c_type}} wscale,
+    {{c_type}}* intercept,
+    {{c_type}}* prediction,
+    int n_classes
+) noexcept nogil:
+    """Compute the prediction given sparse sample x and dense weight w.
+
+    Parameters
+    ----------
+    x_data_ptr : pointer
+        Pointer to the data of the sample x
+
+    x_ind_ptr : pointer
+        Pointer to the indices of the sample  x
+
+    xnnz : int
+        Number of non-zero element in the sample  x
+
+    w_data_ptr : pointer
+        Pointer to the data of the weights w
+
+    wscale : {{c_type}}
+        Scale of the weights w
+
+    intercept : pointer
+        Pointer to the intercept
+
+    prediction : pointer
+        Pointer to store the resulting prediction
+
+    n_classes : int
+        Number of classes in multinomial case. Equals 1 in binary case.
+
+    """
+    cdef int feature_ind, class_ind, j
+    cdef {{c_type}} innerprod
+
+    for class_ind in range(n_classes):
+        innerprod = 0.0
+        # Compute the dot product only on non-zero elements of x
+        for j in range(xnnz):
+            feature_ind = x_ind_ptr[j]
+            innerprod += (w_data_ptr[feature_ind * n_classes + class_ind] *
+                          x_data_ptr[j])
+
+        prediction[class_ind] = wscale * innerprod + intercept[class_ind]
+
+
+{{endfor}}

CameraAI~/Tools~/CWCameraWorker/_internal/sklearn/linear_model/_sgd_fast.pyx.tp

@@ -0,0 +1,661 @@
+{{py:
+
+"""
+Template file to easily generate fused types consistent code using Tempita
+(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
+
+Generated file: _sgd_fast.pyx
+
+Each relevant function is duplicated for the dtypes float and double.
+The keywords between double braces are substituted during the build.
+"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+# The dtypes are defined as follows (name_suffix, c_type, np_type)
+dtypes = [
+    ("64", "double", "np.float64"),
+    ("32", "float", "np.float32"),
+]
+
+}}
+"""SGD implementation"""
+
+import numpy as np
+from time import time
+
+from cython cimport floating
+from libc.math cimport exp, fabs, isfinite, log, pow, INFINITY
+
+from .._loss._loss cimport CyLossFunction
+from ..utils._typedefs cimport uint32_t, uint8_t
+from ..utils._weight_vector cimport WeightVector32, WeightVector64
+from ..utils._seq_dataset cimport SequentialDataset32, SequentialDataset64
+
+
+cdef extern from *:
+    """
+    /* Penalty constants */
+    #define NO_PENALTY 0
+    #define L1 1
+    #define L2 2
+    #define ELASTICNET 3
+
+    /* Learning rate constants */
+    #define CONSTANT 1
+    #define OPTIMAL 2
+    #define INVSCALING 3
+    #define ADAPTIVE 4
+    #define PA1 5
+    #define PA2 6
+    """
+    int NO_PENALTY = 0
+    int L1 = 1
+    int L2 = 2
+    int ELASTICNET = 3
+
+    int CONSTANT = 1
+    int OPTIMAL = 2
+    int INVSCALING = 3
+    int ADAPTIVE = 4
+    int PA1 = 5
+    int PA2 = 6
+
+
+# ----------------------------------------
+# Extension Types for Loss Functions
+# ----------------------------------------
+
+cdef class Regression(CyLossFunction):
+    """Base class for loss functions for regression"""
+
+    def py_loss(self, double p, double y):
+        """Python version of `loss` for testing only.
+
+        Pytest needs a python function and can't use cdef functions.
+
+        Parameters
+        ----------
+        p : double
+            The prediction, `p = w^T x + intercept`.
+        y : double
+            The true value (aka target).
+
+        Returns
+        -------
+        double
+            The loss evaluated at `p` and `y`.
+        """
+        return self.cy_loss(y, p)
+
+    def py_dloss(self, double p, double y):
+        """Python version of `dloss` for testing only.
+
+        Pytest needs a python function and can't use cdef functions.
+
+        Parameters
+        ----------
+        p : double
+            The prediction, `p = w^T x`.
+        y : double
+            The true value (aka target).
+
+        Returns
+        -------
+        double
+            The derivative of the loss function with regards to `p`.
+        """
+        return self.cy_gradient(y, p)
+
+
+cdef class Classification(CyLossFunction):
+    """Base class for loss functions for classification"""
+
+    def py_loss(self, double p, double y):
+        """Python version of `loss` for testing only."""
+        return self.cy_loss(y, p)
+
+    def py_dloss(self, double p, double y):
+        """Python version of `dloss` for testing only."""
+        return self.cy_gradient(y, p)
+
+
+cdef class ModifiedHuber(Classification):
+    """Modified Huber loss for binary classification with y in {-1, 1}
+
+    This is equivalent to quadratically smoothed SVM with gamma = 2.
+
+    See T. Zhang 'Solving Large Scale Linear Prediction Problems Using
+    Stochastic Gradient Descent', ICML'04.
+    """
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
+        cdef double z = p * y
+        if z >= 1.0:
+            return 0.0
+        elif z >= -1.0:
+            return (1.0 - z) * (1.0 - z)
+        else:
+            return -4.0 * z
+
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
+        cdef double z = p * y
+        if z >= 1.0:
+            return 0.0
+        elif z >= -1.0:
+            return 2.0 * (1.0 - z) * -y
+        else:
+            return -4.0 * y
+
+    def __reduce__(self):
+        return ModifiedHuber, ()
+
+
+cdef class Hinge(Classification):
+    """Hinge loss for binary classification tasks with y in {-1,1}
+
+    Parameters
+    ----------
+
+    threshold : float > 0.0
+        Margin threshold. When threshold=1.0, one gets the loss used by SVM.
+        When threshold=0.0, one gets the loss used by the Perceptron.
+    """
+
+    cdef double threshold
+
+    def __init__(self, double threshold=1.0):
+        self.threshold = threshold
+
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
+        cdef double z = p * y
+        if z <= self.threshold:
+            return self.threshold - z
+        return 0.0
+
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
+        cdef double z = p * y
+        if z <= self.threshold:
+            return -y
+        return 0.0
+
+    def __reduce__(self):
+        return Hinge, (self.threshold,)
+
+
+cdef class SquaredHinge(Classification):
+    """Squared Hinge loss for binary classification tasks with y in {-1,1}
+
+    Parameters
+    ----------
+
+    threshold : float > 0.0
+        Margin threshold. When threshold=1.0, one gets the loss used by
+        (quadratically penalized) SVM.
+    """
+
+    cdef double threshold
+
+    def __init__(self, double threshold=1.0):
+        self.threshold = threshold
+
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
+        cdef double z = self.threshold - p * y
+        if z > 0:
+            return z * z
+        return 0.0
+
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
+        cdef double z = self.threshold - p * y
+        if z > 0:
+            return -2 * y * z
+        return 0.0
+
+    def __reduce__(self):
+        return SquaredHinge, (self.threshold,)
+
+
+cdef class EpsilonInsensitive(Regression):
+    """Epsilon-Insensitive loss (used by SVR).
+
+    loss = max(0, |y - p| - epsilon)
+    """
+
+    cdef double epsilon
+
+    def __init__(self, double epsilon):
+        self.epsilon = epsilon
+
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
+        cdef double ret = fabs(y - p) - self.epsilon
+        return ret if ret > 0 else 0
+
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
+        if y - p > self.epsilon:
+            return -1
+        elif p - y > self.epsilon:
+            return 1
+        else:
+            return 0
+
+    def __reduce__(self):
+        return EpsilonInsensitive, (self.epsilon,)
+
+
+cdef class SquaredEpsilonInsensitive(Regression):
+    """Epsilon-Insensitive loss.
+
+    loss = max(0, |y - p| - epsilon)^2
+    """
+
+    cdef double epsilon
+
+    def __init__(self, double epsilon):
+        self.epsilon = epsilon
+
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
+        cdef double ret = fabs(y - p) - self.epsilon
+        return ret * ret if ret > 0 else 0
+
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
+        cdef double z
+        z = y - p
+        if z > self.epsilon:
+            return -2 * (z - self.epsilon)
+        elif z < -self.epsilon:
+            return 2 * (-z - self.epsilon)
+        else:
+            return 0
+
+    def __reduce__(self):
+        return SquaredEpsilonInsensitive, (self.epsilon,)
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+def _plain_sgd{{name_suffix}}(
+    const {{c_type}}[::1] weights,
+    double intercept,
+    const {{c_type}}[::1] average_weights,
+    double average_intercept,
+    CyLossFunction loss,
+    int penalty_type,
+    double alpha,
+    double C,
+    double l1_ratio,
+    SequentialDataset{{name_suffix}} dataset,
+    const uint8_t[::1] validation_mask,
+    bint early_stopping,
+    validation_score_cb,
+    int n_iter_no_change,
+    unsigned int max_iter,
+    double tol,
+    int fit_intercept,
+    int verbose,
+    bint shuffle,
+    uint32_t seed,
+    double weight_pos,
+    double weight_neg,
+    int learning_rate,
+    double eta0,
+    double power_t,
+    bint one_class,
+    double t=1.0,
+    double intercept_decay=1.0,
+    int average=0,
+):
+    """SGD for generic loss functions and penalties with optional averaging
+
+    Parameters
+    ----------
+    weights : ndarray[{{c_type}}, ndim=1]
+        The allocated vector of weights.
+    intercept : double
+        The initial intercept.
+    average_weights : ndarray[{{c_type}}, ndim=1]
+        The average weights as computed for ASGD. Should be None if average
+        is 0.
+    average_intercept : double
+        The average intercept for ASGD. Should be 0 if average is 0.
+    loss : CyLossFunction
+        A concrete ``CyLossFunction`` object.
+    penalty_type : int
+        The penalty 2 for L2, 1 for L1, and 3 for Elastic-Net.
+    alpha : float
+        The regularization parameter.
+    C : float
+        Maximum step size for passive aggressive.
+    l1_ratio : float
+        The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1.
+        l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
+    dataset : SequentialDataset
+        A concrete ``SequentialDataset`` object.
+    validation_mask : ndarray[uint8_t, ndim=1]
+        Equal to True on the validation set.
+    early_stopping : boolean
+        Whether to use a stopping criterion based on the validation set.
+    validation_score_cb : callable
+        A callable to compute a validation score given the current
+        coefficients and intercept values.
+        Used only if early_stopping is True.
+    n_iter_no_change : int
+        Number of iteration with no improvement to wait before stopping.
+    max_iter : int
+        The maximum number of iterations (epochs).
+    tol: double
+        The tolerance for the stopping criterion.
+    fit_intercept : int
+        Whether or not to fit the intercept (1 or 0).
+    verbose : int
+        Print verbose output; 0 for quite.
+    shuffle : boolean
+        Whether to shuffle the training data before each epoch.
+    weight_pos : float
+        The weight of the positive class.
+    weight_neg : float
+        The weight of the negative class.
+    seed : uint32_t
+        Seed of the pseudorandom number generator used to shuffle the data.
+    learning_rate : int
+        The learning rate:
+        (1) constant, eta = eta0
+        (2) optimal, eta = 1.0/(alpha * t).
+        (3) inverse scaling, eta = eta0 / pow(t, power_t)
+        (4) adaptive decrease
+        (5) Passive Aggressive-I, eta = min(alpha, loss/norm(x))
+        (6) Passive Aggressive-II, eta = 1.0 / (norm(x) + 0.5*alpha)
+    eta0 : double
+        The initial learning rate.
+    power_t : double
+        The exponent for inverse scaling learning rate.
+    one_class : boolean
+        Whether to solve the One-Class SVM optimization problem.
+    t : double
+        Initial state of the learning rate. This value is equal to the
+        iteration count except when the learning rate is set to `optimal`.
+        Default: 1.0.
+    average : int
+        The number of iterations before averaging starts. average=1 is
+        equivalent to averaging for all iterations.
+
+
+    Returns
+    -------
+    weights : array, shape=[n_features]
+        The fitted weight vector.
+    intercept : float
+        The fitted intercept term.
+    average_weights : array shape=[n_features]
+        The averaged weights across iterations. Values are valid only if
+        average > 0.
+    average_intercept : float
+        The averaged intercept across iterations.
+        Values are valid only if average > 0.
+    n_iter_ : int
+        The actual number of iter (epochs).
+    """
+
+    # get the data information into easy vars
+    cdef Py_ssize_t n_samples = dataset.n_samples
+    cdef Py_ssize_t n_features = weights.shape[0]
+
+    cdef WeightVector{{name_suffix}} w = WeightVector{{name_suffix}}(weights, average_weights)
+    cdef {{c_type}} *x_data_ptr = NULL
+    cdef int *x_ind_ptr = NULL
+
+    # helper variables
+    cdef int no_improvement_count = 0
+    cdef bint infinity = False
+    cdef int xnnz
+    cdef double eta = 0.0
+    cdef double p = 0.0
+    cdef double update = 0.0
+    cdef double intercept_update = 0.0
+    cdef double sumloss = 0.0
+    cdef double score = 0.0
+    cdef double best_loss = INFINITY
+    cdef double best_score = -INFINITY
+    cdef {{c_type}} y = 0.0
+    cdef {{c_type}} sample_weight
+    cdef {{c_type}} class_weight = 1.0
+    cdef unsigned int count = 0
+    cdef unsigned int train_count = n_samples - np.sum(validation_mask)
+    cdef unsigned int epoch = 0
+    cdef unsigned int i = 0
+    cdef int is_hinge = isinstance(loss, Hinge)
+    cdef double optimal_init = 0.0
+    cdef double dloss = 0.0
+    cdef double MAX_DLOSS = 1e12
+
+    cdef long long sample_index
+
+    # q vector is only used for L1 regularization
+    cdef {{c_type}}[::1] q = None
+    cdef {{c_type}} * q_data_ptr = NULL
+    if penalty_type == L1 or penalty_type == ELASTICNET:
+        q = np.zeros((n_features,), dtype={{np_type}}, order="c")
+        q_data_ptr = &q[0]
+    cdef double u = 0.0
+
+    if penalty_type == L2:
+        l1_ratio = 0.0
+    elif penalty_type == L1:
+        l1_ratio = 1.0
+
+    eta = eta0
+
+    if learning_rate == OPTIMAL:
+        typw = np.sqrt(1.0 / np.sqrt(alpha))
+        # computing eta0, the initial learning rate
+        initial_eta0 = typw / max(1.0, loss.cy_gradient(1.0, -typw))
+        # initialize t such that eta at first sample equals eta0
+        optimal_init = 1.0 / (initial_eta0 * alpha)
+
+    t_start = time()
+    with nogil:
+        for epoch in range(max_iter):
+            sumloss = 0
+            if verbose > 0:
+                with gil:
+                    print("-- Epoch %d" % (epoch + 1))
+            if shuffle:
+                dataset.shuffle(seed)
+            for i in range(n_samples):
+                dataset.next(&x_data_ptr, &x_ind_ptr, &xnnz,
+                             &y, &sample_weight)
+
+                sample_index = dataset.index_data_ptr[dataset.current_index]
+                if validation_mask[sample_index]:
+                    # do not learn on the validation set
+                    continue
+
+                p = w.dot(x_data_ptr, x_ind_ptr, xnnz) + intercept
+                if learning_rate == OPTIMAL:
+                    eta = 1.0 / (alpha * (optimal_init + t - 1))
+                elif learning_rate == INVSCALING:
+                    eta = eta0 / pow(t, power_t)
+
+                if verbose or not early_stopping:
+                    sumloss += loss.cy_loss(y, p)
+
+                if y > 0.0:
+                    class_weight = weight_pos
+                else:
+                    class_weight = weight_neg
+
+                if learning_rate == PA1:
+                    update = sqnorm(x_data_ptr, x_ind_ptr, xnnz)
+                    if update == 0:
+                        continue
+                    update = min(C, loss.cy_loss(y, p) / update)
+                elif learning_rate == PA2:
+                    update = sqnorm(x_data_ptr, x_ind_ptr, xnnz)
+                    update = loss.cy_loss(y, p) / (update + 0.5 / C)
+                else:
+                    dloss = loss.cy_gradient(y, p)
+                    # clip dloss with large values to avoid numerical
+                    # instabilities
+                    if dloss < -MAX_DLOSS:
+                        dloss = -MAX_DLOSS
+                    elif dloss > MAX_DLOSS:
+                        dloss = MAX_DLOSS
+                    update = -eta * dloss
+
+                if learning_rate >= PA1:
+                    if is_hinge:
+                        # classification
+                        update *= y
+                    elif y - p < 0:
+                        # regression
+                        update *= -1
+
+                update *= class_weight * sample_weight
+
+                if penalty_type >= L2:
+                    # do not scale to negative values when eta or alpha are too
+                    # big: instead set the weights to zero
+                    w.scale(max(0, 1.0 - ((1.0 - l1_ratio) * eta * alpha)))
+
+                if update != 0.0:
+                    w.add(x_data_ptr, x_ind_ptr, xnnz, update)
+                if fit_intercept == 1:
+                    intercept_update = update
+                    if one_class:  # specific for One-Class SVM
+                        intercept_update -= 2. * eta * alpha
+                    if intercept_update != 0:
+                        intercept += intercept_update * intercept_decay
+
+                if 0 < average <= t:
+                    # compute the average for the intercept and update the
+                    # average weights, this is done regardless as to whether
+                    # the update is 0
+
+                    w.add_average(x_data_ptr, x_ind_ptr, xnnz,
+                                  update, (t - average + 1))
+                    average_intercept += ((intercept - average_intercept) /
+                                          (t - average + 1))
+
+                if penalty_type == L1 or penalty_type == ELASTICNET:
+                    u += (l1_ratio * eta * alpha)
+                    l1penalty{{name_suffix}}(w, q_data_ptr, x_ind_ptr, xnnz, u)
+
+                t += 1
+                count += 1
+
+            # report epoch information
+            if verbose > 0:
+                with gil:
+                    print("Norm: %.2f, NNZs: %d, Bias: %.6f, T: %d, "
+                          "Avg. loss: %f"
+                          % (w.norm(), np.nonzero(weights)[0].shape[0],
+                             intercept, count, sumloss / train_count))
+                    print("Total training time: %.2f seconds."
+                          % (time() - t_start))
+
+            # floating-point under-/overflow check.
+            if (not isfinite(intercept) or any_nonfinite(weights)):
+                infinity = True
+                break
+
+            # evaluate the score on the validation set
+            if early_stopping:
+                with gil:
+                    score = validation_score_cb(weights.base, intercept)
+                if tol > -INFINITY and score < best_score + tol:
+                    no_improvement_count += 1
+                else:
+                    no_improvement_count = 0
+                if score > best_score:
+                    best_score = score
+            # or evaluate the loss on the training set
+            else:
+                if tol > -INFINITY and sumloss > best_loss - tol * train_count:
+                    no_improvement_count += 1
+                else:
+                    no_improvement_count = 0
+                if sumloss < best_loss:
+                    best_loss = sumloss
+
+            # if there is no improvement several times in a row
+            if no_improvement_count >= n_iter_no_change:
+                if learning_rate == ADAPTIVE and eta > 1e-6:
+                    eta = eta / 5
+                    no_improvement_count = 0
+                else:
+                    if verbose:
+                        with gil:
+                            print("Convergence after %d epochs took %.2f "
+                                  "seconds" % (epoch + 1, time() - t_start))
+                    break
+
+    if infinity:
+        raise ValueError(("Floating-point under-/overflow occurred at epoch"
+                          " #%d. Scaling input data with StandardScaler or"
+                          " MinMaxScaler might help.") % (epoch + 1))
+
+    w.reset_wscale()
+
+    return (
+        weights.base,
+        intercept,
+        None if average_weights is None else average_weights.base,
+        average_intercept,
+        epoch + 1
+    )
+
+{{endfor}}
+
+
+cdef inline bint any_nonfinite(const floating[::1] w) noexcept nogil:
+    for i in range(w.shape[0]):
+        if not isfinite(w[i]):
+            return True
+    return 0
+
+
+cdef inline double sqnorm(
+    floating * x_data_ptr,
+    int * x_ind_ptr,
+    int xnnz,
+) noexcept nogil:
+    cdef double x_norm = 0.0
+    cdef int j
+    cdef double z
+    for j in range(xnnz):
+        z = x_data_ptr[j]
+        x_norm += z * z
+    return x_norm
+
+
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef void l1penalty{{name_suffix}}(
+    WeightVector{{name_suffix}} w,
+    {{c_type}} * q_data_ptr,
+    int *x_ind_ptr,
+    int xnnz,
+    double u,
+) noexcept nogil:
+    """Apply the L1 penalty to each updated feature
+
+    This implements the truncated gradient approach by
+    [Tsuruoka, Y., Tsujii, J., and Ananiadou, S., 2009].
+    """
+    cdef double z = 0.0
+    cdef int j = 0
+    cdef int idx = 0
+    cdef double wscale = w.wscale
+    cdef {{c_type}} *w_data_ptr = w.w_data_ptr
+    for j in range(xnnz):
+        idx = x_ind_ptr[j]
+        z = w_data_ptr[idx]
+        if wscale * z > 0.0:
+            w_data_ptr[idx] = max(
+                0.0, w_data_ptr[idx] - ((u + q_data_ptr[idx]) / wscale))
+
+        elif wscale * z < 0.0:
+            w_data_ptr[idx] = min(
+                0.0, w_data_ptr[idx] + ((u - q_data_ptr[idx]) / wscale))
+
+        q_data_ptr[idx] += wscale * (w_data_ptr[idx] - z)
+
+{{endfor}}

CameraAI~/Tools~/CWCameraWorker/_internal/sklearn/linear_model/meson.build

@@ -0,0 +1,32 @@
+# .pyx is generated, so this is needed to make Cython compilation work
+linear_model_cython_tree = [
+  fs.copyfile('__init__.py'),
+]
+
+py.extension_module(
+  '_cd_fast',
+  [cython_gen.process('_cd_fast.pyx'), utils_cython_tree],
+  subdir: 'sklearn/linear_model',
+  install: true
+)
+
+name_list = ['_sgd_fast', '_sag_fast']
+
+foreach name: name_list
+  pyx = custom_target(
+    name + '_pyx',
+    output: name + '.pyx',
+    input: name + '.pyx.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+    # TODO in principle this should go in py.exension_module below. This is
+    # temporary work-around for dependency issue with .pyx.tp files. For more
+    # details, see https://github.com/mesonbuild/meson/issues/13212
+    depends: [linear_model_cython_tree, utils_cython_tree, _loss_cython_tree],
+  )
+  py.extension_module(
+    name,
+    cython_gen.process(pyx),
+    subdir: 'sklearn/linear_model',
+    install: true
+)
+endforeach