release(camera-ai): publish 0.4.12 catalog portability fix
This commit is contained in:
KINDNICK
@@ -1,268 +1,268 @@
|
||||
from libc.math cimport sqrt, exp
|
||||
|
||||
from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
cdef class DistanceMetric:
|
||||
pass
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist64(
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist64(
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist64(const float64_t dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist64(const float64_t dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric64 base class
|
||||
cdef class DistanceMetric64(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef float64_t dist(
|
||||
self,
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t rdist(
|
||||
self,
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t dist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float64_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t rdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float64_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const float64_t[:, ::1] X,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const float64_t[:, ::1] X,
|
||||
const float64_t[:, ::1] Y,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const float64_t* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t _rdist_to_dist(self, float64_t rdist) except -1 nogil
|
||||
|
||||
cdef float64_t _dist_to_rdist(self, float64_t dist) except -1 nogil
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist32(
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist32(
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist32(const float32_t dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist32(const float32_t dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric32 base class
|
||||
cdef class DistanceMetric32(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef float32_t dist(
|
||||
self,
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t rdist(
|
||||
self,
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t dist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float32_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t rdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float32_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const float32_t[:, ::1] X,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const float32_t[:, ::1] X,
|
||||
const float32_t[:, ::1] Y,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const float32_t* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t _rdist_to_dist(self, float32_t rdist) except -1 nogil
|
||||
|
||||
cdef float32_t _dist_to_rdist(self, float32_t dist) except -1 nogil
|
||||
from libc.math cimport sqrt, exp
|
||||
|
||||
from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
cdef class DistanceMetric:
|
||||
pass
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist64(
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist64(
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist64(const float64_t dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist64(const float64_t dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric64 base class
|
||||
cdef class DistanceMetric64(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef float64_t dist(
|
||||
self,
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t rdist(
|
||||
self,
|
||||
const float64_t* x1,
|
||||
const float64_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t dist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float64_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t rdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float64_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const float64_t[:, ::1] X,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const float64_t[:, ::1] X,
|
||||
const float64_t[:, ::1] Y,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const float64_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const float64_t* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
float64_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float64_t _rdist_to_dist(self, float64_t rdist) except -1 nogil
|
||||
|
||||
cdef float64_t _dist_to_rdist(self, float64_t dist) except -1 nogil
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist32(
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist32(
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist32(const float32_t dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist32(const float32_t dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric32 base class
|
||||
cdef class DistanceMetric32(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef float32_t dist(
|
||||
self,
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t rdist(
|
||||
self,
|
||||
const float32_t* x1,
|
||||
const float32_t* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t dist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float32_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t rdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const float32_t* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const float32_t[:, ::1] X,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const float32_t[:, ::1] X,
|
||||
const float32_t[:, ::1] Y,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const float32_t* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const float32_t* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
float32_t[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef float32_t _rdist_to_dist(self, float32_t rdist) except -1 nogil
|
||||
|
||||
cdef float32_t _dist_to_rdist(self, float32_t dist) except -1 nogil
|
||||
|
||||
@@ -1,152 +1,152 @@
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'float64_t', 'np.float64'),
|
||||
('32', 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
from libc.math cimport sqrt, exp
|
||||
|
||||
from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
cdef class DistanceMetric:
|
||||
pass
|
||||
|
||||
{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist{{name_suffix}}(
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist{{name_suffix}}(
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric{{name_suffix}} base class
|
||||
cdef class DistanceMetric{{name_suffix}}(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} dist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} rdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} dist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} rdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil
|
||||
|
||||
{{endfor}}
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'float64_t', 'np.float64'),
|
||||
('32', 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
from libc.math cimport sqrt, exp
|
||||
|
||||
from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
cdef class DistanceMetric:
|
||||
pass
|
||||
|
||||
{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
######################################################################
|
||||
# Inline distance functions
|
||||
#
|
||||
# We use these for the default (euclidean) case so that they can be
|
||||
# inlined. This leads to faster computation for the most common case
|
||||
cdef inline float64_t euclidean_dist{{name_suffix}}(
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t> (x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return sqrt(d)
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist{{name_suffix}}(
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil:
|
||||
cdef float64_t tmp, d=0
|
||||
cdef intp_t j
|
||||
for j in range(size):
|
||||
tmp = <float64_t>(x1[j] - x2[j])
|
||||
d += tmp * tmp
|
||||
return d
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_dist_to_rdist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
|
||||
return dist * dist
|
||||
|
||||
|
||||
cdef inline float64_t euclidean_rdist_to_dist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
|
||||
return sqrt(dist)
|
||||
|
||||
|
||||
######################################################################
|
||||
# DistanceMetric{{name_suffix}} base class
|
||||
cdef class DistanceMetric{{name_suffix}}(DistanceMetric):
|
||||
# The following attributes are required for a few of the subclasses.
|
||||
# we must define them here so that cython's limited polymorphism will work.
|
||||
# Because we don't expect to instantiate a lot of these objects, the
|
||||
# extra memory overhead of this setup should not be an issue.
|
||||
cdef float64_t p
|
||||
cdef const float64_t[::1] vec
|
||||
cdef const float64_t[:, ::1] mat
|
||||
cdef intp_t size
|
||||
cdef object func
|
||||
cdef object kwargs
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} dist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} rdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1,
|
||||
const {{INPUT_DTYPE_t}}* x2,
|
||||
intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} dist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} rdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t* x1_indices,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t* x2_indices,
|
||||
const int32_t x1_start,
|
||||
const int32_t x1_end,
|
||||
const int32_t x2_start,
|
||||
const int32_t x2_end,
|
||||
const intp_t size,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int pdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int cdist(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1
|
||||
|
||||
cdef int pdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const intp_t size,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef int cdist_csr(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}* x1_data,
|
||||
const int32_t[::1] x1_indices,
|
||||
const int32_t[::1] x1_indptr,
|
||||
const {{INPUT_DTYPE_t}}* x2_data,
|
||||
const int32_t[::1] x2_indices,
|
||||
const int32_t[::1] x2_indptr,
|
||||
const intp_t size,
|
||||
{{INPUT_DTYPE_t}}[:, ::1] D,
|
||||
) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil
|
||||
|
||||
cdef {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+2811
-2811
File diff suppressed because it is too large
Load Diff
+31
-31
@@ -1,31 +1,31 @@
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._base cimport BaseDistancesReduction{{name_suffix}}
|
||||
from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
|
||||
|
||||
cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
|
||||
"""float{{name_suffix}} implementation of the ArgKmin."""
|
||||
|
||||
cdef:
|
||||
intp_t k
|
||||
|
||||
intp_t[:, ::1] argkmin_indices
|
||||
float64_t[:, ::1] argkmin_distances
|
||||
|
||||
# Used as array of pointers to private datastructures used in threads.
|
||||
float64_t ** heaps_r_distances_chunks
|
||||
intp_t ** heaps_indices_chunks
|
||||
|
||||
|
||||
cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
|
||||
"""EuclideanDistance-specialisation of ArgKmin{{name_suffix}}."""
|
||||
cdef:
|
||||
MiddleTermComputer{{name_suffix}} middle_term_computer
|
||||
const float64_t[::1] X_norm_squared
|
||||
const float64_t[::1] Y_norm_squared
|
||||
|
||||
bint use_squared_distances
|
||||
|
||||
{{endfor}}
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._base cimport BaseDistancesReduction{{name_suffix}}
|
||||
from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
|
||||
|
||||
cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
|
||||
"""float{{name_suffix}} implementation of the ArgKmin."""
|
||||
|
||||
cdef:
|
||||
intp_t k
|
||||
|
||||
intp_t[:, ::1] argkmin_indices
|
||||
float64_t[:, ::1] argkmin_distances
|
||||
|
||||
# Used as array of pointers to private datastructures used in threads.
|
||||
float64_t ** heaps_r_distances_chunks
|
||||
intp_t ** heaps_indices_chunks
|
||||
|
||||
|
||||
cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
|
||||
"""EuclideanDistance-specialisation of ArgKmin{{name_suffix}}."""
|
||||
cdef:
|
||||
MiddleTermComputer{{name_suffix}} middle_term_computer
|
||||
const float64_t[::1] X_norm_squared
|
||||
const float64_t[::1] Y_norm_squared
|
||||
|
||||
bint use_squared_distances
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+512
-512
File diff suppressed because it is too large
Load Diff
+182
-182
@@ -1,182 +1,182 @@
|
||||
from cython cimport floating, integral
|
||||
from cython.parallel cimport parallel, prange
|
||||
from libcpp.map cimport map as cpp_map, pair as cpp_pair
|
||||
from libc.stdlib cimport free
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
from ...utils.parallel import _get_threadpool_controller
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import issparse
|
||||
from ._classmode cimport WeightingStrategy
|
||||
|
||||
{{for name_suffix in ["32", "64"]}}
|
||||
from ._argkmin cimport ArgKmin{{name_suffix}}
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
cdef class ArgKminClassMode{{name_suffix}}(ArgKmin{{name_suffix}}):
|
||||
"""
|
||||
{{name_suffix}}bit implementation of ArgKminClassMode.
|
||||
"""
|
||||
cdef:
|
||||
const intp_t[:] Y_labels,
|
||||
const intp_t[:] unique_Y_labels
|
||||
float64_t[:, :] class_scores
|
||||
cpp_map[intp_t, intp_t] labels_to_index
|
||||
WeightingStrategy weight_type
|
||||
|
||||
@classmethod
|
||||
def compute(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
intp_t k,
|
||||
weights,
|
||||
Y_labels,
|
||||
unique_Y_labels,
|
||||
str metric="euclidean",
|
||||
chunk_size=None,
|
||||
dict metric_kwargs=None,
|
||||
str strategy=None,
|
||||
):
|
||||
"""Compute the argkmin reduction with Y_labels.
|
||||
|
||||
This classmethod is responsible for introspecting the arguments
|
||||
values to dispatch to the most appropriate implementation of
|
||||
:class:`ArgKminClassMode{{name_suffix}}`.
|
||||
|
||||
This allows decoupling the API entirely from the implementation details
|
||||
whilst maintaining RAII: all temporarily allocated datastructures necessary
|
||||
for the concrete implementation are therefore freed when this classmethod
|
||||
returns.
|
||||
|
||||
No instance _must_ directly be created outside of this class method.
|
||||
"""
|
||||
# Use a generic implementation that handles most scipy
|
||||
# metrics by computing the distances between 2 vectors at a time.
|
||||
pda = ArgKminClassMode{{name_suffix}}(
|
||||
datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
|
||||
k=k,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
weights=weights,
|
||||
Y_labels=Y_labels,
|
||||
unique_Y_labels=unique_Y_labels,
|
||||
)
|
||||
|
||||
# Limit the number of threads in second level of nested parallelism for BLAS
|
||||
# to avoid threads over-subscription (in GEMM for instance).
|
||||
with _get_threadpool_controller().limit(limits=1, user_api="blas"):
|
||||
if pda.execute_in_parallel_on_Y:
|
||||
pda._parallel_on_Y()
|
||||
else:
|
||||
pda._parallel_on_X()
|
||||
|
||||
return pda._finalize_results()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
DatasetsPair{{name_suffix}} datasets_pair,
|
||||
const intp_t[:] Y_labels,
|
||||
const intp_t[:] unique_Y_labels,
|
||||
chunk_size=None,
|
||||
strategy=None,
|
||||
intp_t k=1,
|
||||
weights=None,
|
||||
):
|
||||
super().__init__(
|
||||
datasets_pair=datasets_pair,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
k=k,
|
||||
)
|
||||
|
||||
if weights == "uniform":
|
||||
self.weight_type = WeightingStrategy.uniform
|
||||
elif weights == "distance":
|
||||
self.weight_type = WeightingStrategy.distance
|
||||
else:
|
||||
self.weight_type = WeightingStrategy.callable
|
||||
self.Y_labels = Y_labels
|
||||
|
||||
self.unique_Y_labels = unique_Y_labels
|
||||
|
||||
cdef intp_t idx, neighbor_class_idx
|
||||
# Map from set of unique labels to their indices in `class_scores`
|
||||
# Buffer used in building a histogram for one-pass weighted mode
|
||||
self.class_scores = np.zeros(
|
||||
(self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
|
||||
)
|
||||
|
||||
def _finalize_results(self):
|
||||
probabilities = np.asarray(self.class_scores)
|
||||
probabilities /= probabilities.sum(axis=1, keepdims=True)
|
||||
return probabilities
|
||||
|
||||
cdef inline void weighted_histogram_mode(
|
||||
self,
|
||||
intp_t sample_index,
|
||||
intp_t* indices,
|
||||
float64_t* distances,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t neighbor_idx, neighbor_class_idx, label_index, multi_output_index
|
||||
float64_t score_incr = 1
|
||||
# TODO: Implement other WeightingStrategy values
|
||||
bint use_distance_weighting = (
|
||||
self.weight_type == WeightingStrategy.distance
|
||||
)
|
||||
|
||||
# Iterate through the sample k-nearest neighbours
|
||||
for neighbor_rank in range(self.k):
|
||||
# Absolute indice of the neighbor_rank-th Nearest Neighbors
|
||||
# in range [0, n_samples_Y)
|
||||
# TODO: inspect if it worth permuting this condition
|
||||
# and the for-loop above for improved branching.
|
||||
if use_distance_weighting:
|
||||
score_incr = 1 / distances[neighbor_rank]
|
||||
neighbor_idx = indices[neighbor_rank]
|
||||
neighbor_class_idx = self.Y_labels[neighbor_idx]
|
||||
self.class_scores[sample_index][neighbor_class_idx] += score_incr
|
||||
return
|
||||
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx, sample_index
|
||||
for idx in range(X_end - X_start):
|
||||
# One-pass top-one weighted mode
|
||||
# Compute the absolute index in [0, n_samples_X)
|
||||
sample_index = X_start + idx
|
||||
self.weighted_histogram_mode(
|
||||
sample_index,
|
||||
&self.heaps_indices_chunks[thread_num][idx * self.k],
|
||||
&self.heaps_r_distances_chunks[thread_num][idx * self.k],
|
||||
)
|
||||
return
|
||||
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t sample_index, thread_num
|
||||
|
||||
with nogil, parallel(num_threads=self.chunks_n_threads):
|
||||
# Deallocating temporary datastructures
|
||||
for thread_num in prange(self.chunks_n_threads, schedule='static'):
|
||||
free(self.heaps_r_distances_chunks[thread_num])
|
||||
free(self.heaps_indices_chunks[thread_num])
|
||||
|
||||
for sample_index in prange(self.n_samples_X, schedule='static'):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index,
|
||||
&self.argkmin_indices[sample_index][0],
|
||||
&self.argkmin_distances[sample_index][0],
|
||||
)
|
||||
return
|
||||
|
||||
{{endfor}}
|
||||
from cython cimport floating, integral
|
||||
from cython.parallel cimport parallel, prange
|
||||
from libcpp.map cimport map as cpp_map, pair as cpp_pair
|
||||
from libc.stdlib cimport free
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
from ...utils.parallel import _get_threadpool_controller
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import issparse
|
||||
from ._classmode cimport WeightingStrategy
|
||||
|
||||
{{for name_suffix in ["32", "64"]}}
|
||||
from ._argkmin cimport ArgKmin{{name_suffix}}
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
cdef class ArgKminClassMode{{name_suffix}}(ArgKmin{{name_suffix}}):
|
||||
"""
|
||||
{{name_suffix}}bit implementation of ArgKminClassMode.
|
||||
"""
|
||||
cdef:
|
||||
const intp_t[:] Y_labels,
|
||||
const intp_t[:] unique_Y_labels
|
||||
float64_t[:, :] class_scores
|
||||
cpp_map[intp_t, intp_t] labels_to_index
|
||||
WeightingStrategy weight_type
|
||||
|
||||
@classmethod
|
||||
def compute(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
intp_t k,
|
||||
weights,
|
||||
Y_labels,
|
||||
unique_Y_labels,
|
||||
str metric="euclidean",
|
||||
chunk_size=None,
|
||||
dict metric_kwargs=None,
|
||||
str strategy=None,
|
||||
):
|
||||
"""Compute the argkmin reduction with Y_labels.
|
||||
|
||||
This classmethod is responsible for introspecting the arguments
|
||||
values to dispatch to the most appropriate implementation of
|
||||
:class:`ArgKminClassMode{{name_suffix}}`.
|
||||
|
||||
This allows decoupling the API entirely from the implementation details
|
||||
whilst maintaining RAII: all temporarily allocated datastructures necessary
|
||||
for the concrete implementation are therefore freed when this classmethod
|
||||
returns.
|
||||
|
||||
No instance _must_ directly be created outside of this class method.
|
||||
"""
|
||||
# Use a generic implementation that handles most scipy
|
||||
# metrics by computing the distances between 2 vectors at a time.
|
||||
pda = ArgKminClassMode{{name_suffix}}(
|
||||
datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
|
||||
k=k,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
weights=weights,
|
||||
Y_labels=Y_labels,
|
||||
unique_Y_labels=unique_Y_labels,
|
||||
)
|
||||
|
||||
# Limit the number of threads in second level of nested parallelism for BLAS
|
||||
# to avoid threads over-subscription (in GEMM for instance).
|
||||
with _get_threadpool_controller().limit(limits=1, user_api="blas"):
|
||||
if pda.execute_in_parallel_on_Y:
|
||||
pda._parallel_on_Y()
|
||||
else:
|
||||
pda._parallel_on_X()
|
||||
|
||||
return pda._finalize_results()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
DatasetsPair{{name_suffix}} datasets_pair,
|
||||
const intp_t[:] Y_labels,
|
||||
const intp_t[:] unique_Y_labels,
|
||||
chunk_size=None,
|
||||
strategy=None,
|
||||
intp_t k=1,
|
||||
weights=None,
|
||||
):
|
||||
super().__init__(
|
||||
datasets_pair=datasets_pair,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
k=k,
|
||||
)
|
||||
|
||||
if weights == "uniform":
|
||||
self.weight_type = WeightingStrategy.uniform
|
||||
elif weights == "distance":
|
||||
self.weight_type = WeightingStrategy.distance
|
||||
else:
|
||||
self.weight_type = WeightingStrategy.callable
|
||||
self.Y_labels = Y_labels
|
||||
|
||||
self.unique_Y_labels = unique_Y_labels
|
||||
|
||||
cdef intp_t idx, neighbor_class_idx
|
||||
# Map from set of unique labels to their indices in `class_scores`
|
||||
# Buffer used in building a histogram for one-pass weighted mode
|
||||
self.class_scores = np.zeros(
|
||||
(self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
|
||||
)
|
||||
|
||||
def _finalize_results(self):
|
||||
probabilities = np.asarray(self.class_scores)
|
||||
probabilities /= probabilities.sum(axis=1, keepdims=True)
|
||||
return probabilities
|
||||
|
||||
cdef inline void weighted_histogram_mode(
|
||||
self,
|
||||
intp_t sample_index,
|
||||
intp_t* indices,
|
||||
float64_t* distances,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t neighbor_idx, neighbor_class_idx, label_index, multi_output_index
|
||||
float64_t score_incr = 1
|
||||
# TODO: Implement other WeightingStrategy values
|
||||
bint use_distance_weighting = (
|
||||
self.weight_type == WeightingStrategy.distance
|
||||
)
|
||||
|
||||
# Iterate through the sample k-nearest neighbours
|
||||
for neighbor_rank in range(self.k):
|
||||
# Absolute indice of the neighbor_rank-th Nearest Neighbors
|
||||
# in range [0, n_samples_Y)
|
||||
# TODO: inspect if it worth permuting this condition
|
||||
# and the for-loop above for improved branching.
|
||||
if use_distance_weighting:
|
||||
score_incr = 1 / distances[neighbor_rank]
|
||||
neighbor_idx = indices[neighbor_rank]
|
||||
neighbor_class_idx = self.Y_labels[neighbor_idx]
|
||||
self.class_scores[sample_index][neighbor_class_idx] += score_incr
|
||||
return
|
||||
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx, sample_index
|
||||
for idx in range(X_end - X_start):
|
||||
# One-pass top-one weighted mode
|
||||
# Compute the absolute index in [0, n_samples_X)
|
||||
sample_index = X_start + idx
|
||||
self.weighted_histogram_mode(
|
||||
sample_index,
|
||||
&self.heaps_indices_chunks[thread_num][idx * self.k],
|
||||
&self.heaps_r_distances_chunks[thread_num][idx * self.k],
|
||||
)
|
||||
return
|
||||
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t sample_index, thread_num
|
||||
|
||||
with nogil, parallel(num_threads=self.chunks_n_threads):
|
||||
# Deallocating temporary datastructures
|
||||
for thread_num in prange(self.chunks_n_threads, schedule='static'):
|
||||
free(self.heaps_r_distances_chunks[thread_num])
|
||||
free(self.heaps_indices_chunks[thread_num])
|
||||
|
||||
for sample_index in prange(self.n_samples_X, schedule='static'):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index,
|
||||
&self.argkmin_indices[sample_index][0],
|
||||
&self.argkmin_distances[sample_index][0],
|
||||
)
|
||||
return
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+135
-135
@@ -1,135 +1,135 @@
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
|
||||
cpdef float64_t[::1] _sqeuclidean_row_norms{{name_suffix}}(
|
||||
X,
|
||||
intp_t num_threads,
|
||||
)
|
||||
|
||||
cdef class BaseDistancesReduction{{name_suffix}}:
|
||||
"""
|
||||
Base float{{name_suffix}} implementation template of the pairwise-distances
|
||||
reduction backends.
|
||||
|
||||
Implementations inherit from this template and may override the several
|
||||
defined hooks as needed in order to easily extend functionality with
|
||||
minimal redundant code.
|
||||
"""
|
||||
|
||||
cdef:
|
||||
readonly DatasetsPair{{name_suffix}} datasets_pair
|
||||
|
||||
# The number of threads that can be used is stored in effective_n_threads.
|
||||
#
|
||||
# The number of threads to use in the parallelization strategy
|
||||
# (i.e. parallel_on_X or parallel_on_Y) can be smaller than effective_n_threads:
|
||||
# for small datasets, fewer threads might be needed to loop over pair of chunks.
|
||||
#
|
||||
# Hence, the number of threads that _will_ be used for looping over chunks
|
||||
# is stored in chunks_n_threads, allowing solely using what we need.
|
||||
#
|
||||
# Thus, an invariant is:
|
||||
#
|
||||
# chunks_n_threads <= effective_n_threads
|
||||
#
|
||||
intp_t effective_n_threads
|
||||
intp_t chunks_n_threads
|
||||
|
||||
intp_t n_samples_chunk, chunk_size
|
||||
|
||||
intp_t n_samples_X, X_n_samples_chunk, X_n_chunks, X_n_samples_last_chunk
|
||||
intp_t n_samples_Y, Y_n_samples_chunk, Y_n_chunks, Y_n_samples_last_chunk
|
||||
|
||||
bint execute_in_parallel_on_Y
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_X(self) noexcept nogil
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_Y(self) noexcept nogil
|
||||
|
||||
# Placeholder methods which have to be implemented
|
||||
|
||||
cdef void _compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
# Placeholder methods which can be implemented
|
||||
|
||||
cdef void compute_exact_distances(self) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_finalize(
|
||||
self,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_init(
|
||||
self,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_synchronize(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil
|
||||
{{endfor}}
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
|
||||
cpdef float64_t[::1] _sqeuclidean_row_norms{{name_suffix}}(
|
||||
X,
|
||||
intp_t num_threads,
|
||||
)
|
||||
|
||||
cdef class BaseDistancesReduction{{name_suffix}}:
|
||||
"""
|
||||
Base float{{name_suffix}} implementation template of the pairwise-distances
|
||||
reduction backends.
|
||||
|
||||
Implementations inherit from this template and may override the several
|
||||
defined hooks as needed in order to easily extend functionality with
|
||||
minimal redundant code.
|
||||
"""
|
||||
|
||||
cdef:
|
||||
readonly DatasetsPair{{name_suffix}} datasets_pair
|
||||
|
||||
# The number of threads that can be used is stored in effective_n_threads.
|
||||
#
|
||||
# The number of threads to use in the parallelization strategy
|
||||
# (i.e. parallel_on_X or parallel_on_Y) can be smaller than effective_n_threads:
|
||||
# for small datasets, fewer threads might be needed to loop over pair of chunks.
|
||||
#
|
||||
# Hence, the number of threads that _will_ be used for looping over chunks
|
||||
# is stored in chunks_n_threads, allowing solely using what we need.
|
||||
#
|
||||
# Thus, an invariant is:
|
||||
#
|
||||
# chunks_n_threads <= effective_n_threads
|
||||
#
|
||||
intp_t effective_n_threads
|
||||
intp_t chunks_n_threads
|
||||
|
||||
intp_t n_samples_chunk, chunk_size
|
||||
|
||||
intp_t n_samples_X, X_n_samples_chunk, X_n_chunks, X_n_samples_last_chunk
|
||||
intp_t n_samples_Y, Y_n_samples_chunk, Y_n_chunks, Y_n_samples_last_chunk
|
||||
|
||||
bint execute_in_parallel_on_Y
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_X(self) noexcept nogil
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_Y(self) noexcept nogil
|
||||
|
||||
# Placeholder methods which have to be implemented
|
||||
|
||||
cdef void _compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
# Placeholder methods which can be implemented
|
||||
|
||||
cdef void compute_exact_distances(self) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_finalize(
|
||||
self,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_init(
|
||||
self,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_synchronize(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil
|
||||
{{endfor}}
|
||||
|
||||
+504
-504
File diff suppressed because it is too large
Load Diff
+5
-5
@@ -1,5 +1,5 @@
|
||||
cpdef enum WeightingStrategy:
|
||||
uniform = 0
|
||||
# TODO: Implement the following options in weighted_histogram_mode
|
||||
distance = 1
|
||||
callable = 2
|
||||
cpdef enum WeightingStrategy:
|
||||
uniform = 0
|
||||
# TODO: Implement the following options in weighted_histogram_mode
|
||||
distance = 1
|
||||
callable = 2
|
||||
|
||||
+67
-67
@@ -1,67 +1,67 @@
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'DistanceMetric64', 'float64_t'),
|
||||
('32', 'DistanceMetric32', 'float32_t')
|
||||
]
|
||||
|
||||
}}
|
||||
from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
from ...metrics._dist_metrics cimport DistanceMetric64, DistanceMetric32, DistanceMetric
|
||||
|
||||
{{for name_suffix, DistanceMetric, INPUT_DTYPE_t in implementation_specific_values}}
|
||||
|
||||
|
||||
cdef class DatasetsPair{{name_suffix}}:
|
||||
cdef:
|
||||
{{DistanceMetric}} distance_metric
|
||||
intp_t n_features
|
||||
|
||||
cdef intp_t n_samples_X(self) noexcept nogil
|
||||
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil
|
||||
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil
|
||||
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil
|
||||
|
||||
|
||||
cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
|
||||
cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:] X_data
|
||||
const int32_t[::1] X_indices
|
||||
const int32_t[::1] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:] Y_data
|
||||
const int32_t[::1] Y_indices
|
||||
const int32_t[::1] Y_indptr
|
||||
|
||||
|
||||
cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:] X_data
|
||||
const int32_t[::1] X_indices
|
||||
const int32_t[::1] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:] Y_data
|
||||
const int32_t[::1] Y_indices
|
||||
intp_t n_Y
|
||||
|
||||
|
||||
cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
# As distance metrics are commutative, we can simply rely
|
||||
# on the implementation of SparseDenseDatasetsPair and
|
||||
# swap arguments.
|
||||
DatasetsPair{{name_suffix}} datasets_pair
|
||||
|
||||
{{endfor}}
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'DistanceMetric64', 'float64_t'),
|
||||
('32', 'DistanceMetric32', 'float32_t')
|
||||
]
|
||||
|
||||
}}
|
||||
from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
from ...metrics._dist_metrics cimport DistanceMetric64, DistanceMetric32, DistanceMetric
|
||||
|
||||
{{for name_suffix, DistanceMetric, INPUT_DTYPE_t in implementation_specific_values}}
|
||||
|
||||
|
||||
cdef class DatasetsPair{{name_suffix}}:
|
||||
cdef:
|
||||
{{DistanceMetric}} distance_metric
|
||||
intp_t n_features
|
||||
|
||||
cdef intp_t n_samples_X(self) noexcept nogil
|
||||
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil
|
||||
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil
|
||||
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil
|
||||
|
||||
|
||||
cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
|
||||
cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:] X_data
|
||||
const int32_t[::1] X_indices
|
||||
const int32_t[::1] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:] Y_data
|
||||
const int32_t[::1] Y_indices
|
||||
const int32_t[::1] Y_indptr
|
||||
|
||||
|
||||
cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:] X_data
|
||||
const int32_t[::1] X_indices
|
||||
const int32_t[::1] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:] Y_data
|
||||
const int32_t[::1] Y_indices
|
||||
intp_t n_Y
|
||||
|
||||
|
||||
cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
cdef:
|
||||
# As distance metrics are commutative, we can simply rely
|
||||
# on the implementation of SparseDenseDatasetsPair and
|
||||
# swap arguments.
|
||||
DatasetsPair{{name_suffix}} datasets_pair
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+406
-406
@@ -1,406 +1,406 @@
|
||||
import copy
|
||||
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'DistanceMetric64', 'float64_t', 'np.float64'),
|
||||
('32', 'DistanceMetric32', 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
import numpy as np
|
||||
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport float64_t, float32_t, intp_t
|
||||
|
||||
from scipy.sparse import issparse, csr_matrix
|
||||
|
||||
{{for name_suffix, DistanceMetric, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
cdef class DatasetsPair{{name_suffix}}:
|
||||
"""Abstract class which wraps a pair of datasets (X, Y).
|
||||
|
||||
This class allows computing distances between a single pair of rows of
|
||||
of X and Y at a time given the pair of their indices (i, j). This class is
|
||||
specialized for each metric thanks to the :func:`get_for` factory classmethod.
|
||||
|
||||
The handling of parallelization over chunks to compute the distances
|
||||
and aggregation for several rows at a time is done in dedicated
|
||||
subclasses of :class:`BaseDistancesReductionDispatcher` that in-turn rely on
|
||||
subclasses of :class:`DatasetsPair` for each pair of rows in the data. The
|
||||
goal is to make it possible to decouple the generic parallelization and
|
||||
aggregation logic from metric-specific computation as much as possible.
|
||||
|
||||
X and Y can be stored as C-contiguous np.ndarrays or CSR matrices
|
||||
in subclasses.
|
||||
|
||||
This class avoids the overhead of dispatching distance computations
|
||||
to :class:`sklearn.metrics.DistanceMetric` based on the physical
|
||||
representation of the vectors (sparse vs. dense). It makes use of
|
||||
cython.final to remove the overhead of dispatching method calls.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
distance_metric: {{DistanceMetric}}
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_for(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
metric="euclidean",
|
||||
dict metric_kwargs=None,
|
||||
) -> DatasetsPair{{name_suffix}}:
|
||||
"""Return the DatasetsPair implementation for the given arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : {ndarray, sparse matrix} of shape (n_samples_X, n_features)
|
||||
Input data.
|
||||
If provided as a ndarray, it must be C-contiguous.
|
||||
If provided as a sparse matrix, it must be in CSR format.
|
||||
|
||||
Y : {ndarray, sparse matrix} of shape (n_samples_Y, n_features)
|
||||
Input data.
|
||||
If provided as a ndarray, it must be C-contiguous.
|
||||
If provided as a sparse matrix, it must be in CSR format.
|
||||
|
||||
metric : str or DistanceMetric object, default='euclidean'
|
||||
The distance metric to compute between rows of X and Y.
|
||||
The default metric is a fast implementation of the Euclidean
|
||||
metric. For a list of available metrics, see the documentation
|
||||
of :class:`~sklearn.metrics.DistanceMetric`.
|
||||
|
||||
metric_kwargs : dict, default=None
|
||||
Keyword arguments to pass to specified metric function.
|
||||
|
||||
Returns
|
||||
-------
|
||||
datasets_pair: DatasetsPair{{name_suffix}}
|
||||
The suited DatasetsPair{{name_suffix}} implementation.
|
||||
"""
|
||||
# X_norm_squared and Y_norm_squared might be propagated
|
||||
# down to DatasetsPairs via metrics_kwargs when the Euclidean
|
||||
# specialisations can't be used.
|
||||
# To prevent X_norm_squared and Y_norm_squared to be passed
|
||||
# down to DistanceMetrics (whose constructors would raise
|
||||
# a RuntimeError), we pop them here.
|
||||
if metric_kwargs is not None:
|
||||
# Copying metric_kwargs not to pop "X_norm_squared"
|
||||
# and "Y_norm_squared" where they are used
|
||||
metric_kwargs = copy.copy(metric_kwargs)
|
||||
metric_kwargs.pop("X_norm_squared", None)
|
||||
metric_kwargs.pop("Y_norm_squared", None)
|
||||
cdef:
|
||||
{{DistanceMetric}} distance_metric = DistanceMetric.get_metric(
|
||||
metric,
|
||||
{{INPUT_DTYPE}},
|
||||
**(metric_kwargs or {})
|
||||
)
|
||||
|
||||
# Metric-specific checks that do not replace nor duplicate `check_array`.
|
||||
distance_metric._validate_data(X)
|
||||
distance_metric._validate_data(Y)
|
||||
|
||||
X_is_sparse = issparse(X)
|
||||
Y_is_sparse = issparse(Y)
|
||||
|
||||
if not X_is_sparse and not Y_is_sparse:
|
||||
return DenseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
if X_is_sparse and Y_is_sparse:
|
||||
return SparseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
if X_is_sparse and not Y_is_sparse:
|
||||
return SparseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
return DenseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
@classmethod
|
||||
def unpack_csr_matrix(cls, X: csr_matrix):
|
||||
"""Ensure that the CSR matrix is indexed with np.int32."""
|
||||
X_data = np.asarray(X.data, dtype={{INPUT_DTYPE}})
|
||||
X_indices = np.asarray(X.indices, dtype=np.int32)
|
||||
X_indptr = np.asarray(X.indptr, dtype=np.int32)
|
||||
return X_data, X_indices, X_indptr
|
||||
|
||||
def __init__(self, {{DistanceMetric}} distance_metric, intp_t n_features):
|
||||
self.distance_metric = distance_metric
|
||||
self.n_features = n_features
|
||||
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
"""Number of samples in X."""
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -999
|
||||
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
"""Number of samples in Y."""
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -999
|
||||
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.dist(i, j)
|
||||
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -1
|
||||
|
||||
@final
|
||||
cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between row vectors of two arrays.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: ndarray of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
Y: ndarray of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two row vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y,
|
||||
{{DistanceMetric}} distance_metric,
|
||||
):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
# Arrays have already been checked
|
||||
self.X = X
|
||||
self.Y = Y
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X.shape[0]
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.Y.shape[0]
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.dist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
|
||||
|
||||
|
||||
@final
|
||||
cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of two CSR matrices.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: sparse matrix of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
Y: sparse matrix of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
|
||||
self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
|
||||
self.Y_data, self.Y_indices, self.Y_indptr = self.unpack_csr_matrix(Y)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.Y_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
x2_data=&self.Y_data[0],
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=self.Y_indptr[j],
|
||||
x2_end=self.Y_indptr[j + 1],
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.dist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
x2_data=&self.Y_data[0],
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=self.Y_indptr[j],
|
||||
x2_end=self.Y_indptr[j + 1],
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
|
||||
@final
|
||||
cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of a CSR matrix and a dense array.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: sparse matrix of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
Y: ndarray of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
|
||||
self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
|
||||
|
||||
# We support the sparse-dense case by using the sparse-sparse interfaces
|
||||
# of `DistanceMetric` (namely `DistanceMetric.{dist_csr,rdist_csr}`) to
|
||||
# avoid introducing a new complex set of interfaces. In this case, we
|
||||
# need to convert `Y` (the dense array) into a CSR matrix.
|
||||
#
|
||||
# Here we motive using another simpler CSR representation to use for `Y`.
|
||||
#
|
||||
# If we were to use the usual CSR representation for `Y`, storing all
|
||||
# the columns indices in `indices` would have required allocating an
|
||||
# array of n_samples × n_features elements with repeated contiguous
|
||||
# integers from 0 to n_features - 1. This would have been very wasteful
|
||||
# from a memory point of view. This alternative representation just uses
|
||||
# the necessary amount of information needed and only necessitates
|
||||
# shifting the address of `data` before calling the CSR × CSR routines.
|
||||
#
|
||||
# In this representation:
|
||||
#
|
||||
# - the `data` array is the original dense array, `Y`, whose first
|
||||
# element's address is shifted before calling the CSR × CSR routine
|
||||
#
|
||||
# - the `indices` array is a single row of `n_features` elements:
|
||||
#
|
||||
# [0, 1, ..., n_features-1]
|
||||
#
|
||||
# - the `indptr` array is not materialised as the indices pointers'
|
||||
# offset is constant (the offset equals `n_features`). Moreover, as
|
||||
# `data` is shifted, constant `start` and `end` indices pointers
|
||||
# respectively equalling 0 and n_features are used.
|
||||
|
||||
# Y array already has been checked here
|
||||
self.n_Y = Y.shape[0]
|
||||
self.Y_data = np.ravel(Y)
|
||||
self.Y_indices = np.arange(self.n_features, dtype=np.int32)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.n_Y
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
# Increment the data pointer such that x2_start=0 is aligned with the
|
||||
# j-th row
|
||||
x2_data=&self.Y_data[0] + j * self.n_features,
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=0,
|
||||
x2_end=self.n_features,
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
|
||||
return self.distance_metric.dist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
# Increment the data pointer such that x2_start=0 is aligned with the
|
||||
# j-th row
|
||||
x2_data=&self.Y_data[0] + j * self.n_features,
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=0,
|
||||
x2_end=self.n_features,
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
|
||||
@final
|
||||
cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of a dense array and a CSR matrix.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: ndarray of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
Y: sparse matrix of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
# Swapping arguments on the constructor
|
||||
self.datasets_pair = SparseDenseDatasetsPair{{name_suffix}}(Y, X, distance_metric)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
# Swapping interface
|
||||
return self.datasets_pair.n_samples_Y()
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
# Swapping interface
|
||||
return self.datasets_pair.n_samples_X()
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# Swapping arguments on the same interface
|
||||
return self.datasets_pair.surrogate_dist(j, i)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# Swapping arguments on the same interface
|
||||
return self.datasets_pair.dist(j, i)
|
||||
|
||||
{{endfor}}
|
||||
import copy
|
||||
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
('64', 'DistanceMetric64', 'float64_t', 'np.float64'),
|
||||
('32', 'DistanceMetric32', 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
import numpy as np
|
||||
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport float64_t, float32_t, intp_t
|
||||
|
||||
from scipy.sparse import issparse, csr_matrix
|
||||
|
||||
{{for name_suffix, DistanceMetric, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
cdef class DatasetsPair{{name_suffix}}:
|
||||
"""Abstract class which wraps a pair of datasets (X, Y).
|
||||
|
||||
This class allows computing distances between a single pair of rows of
|
||||
of X and Y at a time given the pair of their indices (i, j). This class is
|
||||
specialized for each metric thanks to the :func:`get_for` factory classmethod.
|
||||
|
||||
The handling of parallelization over chunks to compute the distances
|
||||
and aggregation for several rows at a time is done in dedicated
|
||||
subclasses of :class:`BaseDistancesReductionDispatcher` that in-turn rely on
|
||||
subclasses of :class:`DatasetsPair` for each pair of rows in the data. The
|
||||
goal is to make it possible to decouple the generic parallelization and
|
||||
aggregation logic from metric-specific computation as much as possible.
|
||||
|
||||
X and Y can be stored as C-contiguous np.ndarrays or CSR matrices
|
||||
in subclasses.
|
||||
|
||||
This class avoids the overhead of dispatching distance computations
|
||||
to :class:`sklearn.metrics.DistanceMetric` based on the physical
|
||||
representation of the vectors (sparse vs. dense). It makes use of
|
||||
cython.final to remove the overhead of dispatching method calls.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
distance_metric: {{DistanceMetric}}
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_for(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
metric="euclidean",
|
||||
dict metric_kwargs=None,
|
||||
) -> DatasetsPair{{name_suffix}}:
|
||||
"""Return the DatasetsPair implementation for the given arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : {ndarray, sparse matrix} of shape (n_samples_X, n_features)
|
||||
Input data.
|
||||
If provided as a ndarray, it must be C-contiguous.
|
||||
If provided as a sparse matrix, it must be in CSR format.
|
||||
|
||||
Y : {ndarray, sparse matrix} of shape (n_samples_Y, n_features)
|
||||
Input data.
|
||||
If provided as a ndarray, it must be C-contiguous.
|
||||
If provided as a sparse matrix, it must be in CSR format.
|
||||
|
||||
metric : str or DistanceMetric object, default='euclidean'
|
||||
The distance metric to compute between rows of X and Y.
|
||||
The default metric is a fast implementation of the Euclidean
|
||||
metric. For a list of available metrics, see the documentation
|
||||
of :class:`~sklearn.metrics.DistanceMetric`.
|
||||
|
||||
metric_kwargs : dict, default=None
|
||||
Keyword arguments to pass to specified metric function.
|
||||
|
||||
Returns
|
||||
-------
|
||||
datasets_pair: DatasetsPair{{name_suffix}}
|
||||
The suited DatasetsPair{{name_suffix}} implementation.
|
||||
"""
|
||||
# X_norm_squared and Y_norm_squared might be propagated
|
||||
# down to DatasetsPairs via metrics_kwargs when the Euclidean
|
||||
# specialisations can't be used.
|
||||
# To prevent X_norm_squared and Y_norm_squared to be passed
|
||||
# down to DistanceMetrics (whose constructors would raise
|
||||
# a RuntimeError), we pop them here.
|
||||
if metric_kwargs is not None:
|
||||
# Copying metric_kwargs not to pop "X_norm_squared"
|
||||
# and "Y_norm_squared" where they are used
|
||||
metric_kwargs = copy.copy(metric_kwargs)
|
||||
metric_kwargs.pop("X_norm_squared", None)
|
||||
metric_kwargs.pop("Y_norm_squared", None)
|
||||
cdef:
|
||||
{{DistanceMetric}} distance_metric = DistanceMetric.get_metric(
|
||||
metric,
|
||||
{{INPUT_DTYPE}},
|
||||
**(metric_kwargs or {})
|
||||
)
|
||||
|
||||
# Metric-specific checks that do not replace nor duplicate `check_array`.
|
||||
distance_metric._validate_data(X)
|
||||
distance_metric._validate_data(Y)
|
||||
|
||||
X_is_sparse = issparse(X)
|
||||
Y_is_sparse = issparse(Y)
|
||||
|
||||
if not X_is_sparse and not Y_is_sparse:
|
||||
return DenseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
if X_is_sparse and Y_is_sparse:
|
||||
return SparseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
if X_is_sparse and not Y_is_sparse:
|
||||
return SparseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
return DenseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
|
||||
|
||||
@classmethod
|
||||
def unpack_csr_matrix(cls, X: csr_matrix):
|
||||
"""Ensure that the CSR matrix is indexed with np.int32."""
|
||||
X_data = np.asarray(X.data, dtype={{INPUT_DTYPE}})
|
||||
X_indices = np.asarray(X.indices, dtype=np.int32)
|
||||
X_indptr = np.asarray(X.indptr, dtype=np.int32)
|
||||
return X_data, X_indices, X_indptr
|
||||
|
||||
def __init__(self, {{DistanceMetric}} distance_metric, intp_t n_features):
|
||||
self.distance_metric = distance_metric
|
||||
self.n_features = n_features
|
||||
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
"""Number of samples in X."""
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -999
|
||||
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
"""Number of samples in Y."""
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -999
|
||||
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.dist(i, j)
|
||||
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# This is a abstract method.
|
||||
# This _must_ always be overwritten in subclasses.
|
||||
# TODO: add "with gil: raise" here when supporting Cython 3.0
|
||||
return -1
|
||||
|
||||
@final
|
||||
cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between row vectors of two arrays.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: ndarray of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
Y: ndarray of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two row vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X,
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y,
|
||||
{{DistanceMetric}} distance_metric,
|
||||
):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
# Arrays have already been checked
|
||||
self.X = X
|
||||
self.Y = Y
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X.shape[0]
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.Y.shape[0]
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.dist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
|
||||
|
||||
|
||||
@final
|
||||
cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of two CSR matrices.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: sparse matrix of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
Y: sparse matrix of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
|
||||
self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
|
||||
self.Y_data, self.Y_indices, self.Y_indptr = self.unpack_csr_matrix(Y)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.Y_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
x2_data=&self.Y_data[0],
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=self.Y_indptr[j],
|
||||
x2_end=self.Y_indptr[j + 1],
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.dist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
x2_data=&self.Y_data[0],
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=self.Y_indptr[j],
|
||||
x2_end=self.Y_indptr[j + 1],
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
|
||||
@final
|
||||
cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of a CSR matrix and a dense array.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: sparse matrix of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
Y: ndarray of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
|
||||
self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
|
||||
|
||||
# We support the sparse-dense case by using the sparse-sparse interfaces
|
||||
# of `DistanceMetric` (namely `DistanceMetric.{dist_csr,rdist_csr}`) to
|
||||
# avoid introducing a new complex set of interfaces. In this case, we
|
||||
# need to convert `Y` (the dense array) into a CSR matrix.
|
||||
#
|
||||
# Here we motive using another simpler CSR representation to use for `Y`.
|
||||
#
|
||||
# If we were to use the usual CSR representation for `Y`, storing all
|
||||
# the columns indices in `indices` would have required allocating an
|
||||
# array of n_samples × n_features elements with repeated contiguous
|
||||
# integers from 0 to n_features - 1. This would have been very wasteful
|
||||
# from a memory point of view. This alternative representation just uses
|
||||
# the necessary amount of information needed and only necessitates
|
||||
# shifting the address of `data` before calling the CSR × CSR routines.
|
||||
#
|
||||
# In this representation:
|
||||
#
|
||||
# - the `data` array is the original dense array, `Y`, whose first
|
||||
# element's address is shifted before calling the CSR × CSR routine
|
||||
#
|
||||
# - the `indices` array is a single row of `n_features` elements:
|
||||
#
|
||||
# [0, 1, ..., n_features-1]
|
||||
#
|
||||
# - the `indptr` array is not materialised as the indices pointers'
|
||||
# offset is constant (the offset equals `n_features`). Moreover, as
|
||||
# `data` is shifted, constant `start` and `end` indices pointers
|
||||
# respectively equalling 0 and n_features are used.
|
||||
|
||||
# Y array already has been checked here
|
||||
self.n_Y = Y.shape[0]
|
||||
self.Y_data = np.ravel(Y)
|
||||
self.Y_indices = np.arange(self.n_features, dtype=np.int32)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
return self.X_indptr.shape[0] - 1
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
return self.n_Y
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
return self.distance_metric.rdist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
# Increment the data pointer such that x2_start=0 is aligned with the
|
||||
# j-th row
|
||||
x2_data=&self.Y_data[0] + j * self.n_features,
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=0,
|
||||
x2_end=self.n_features,
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
|
||||
return self.distance_metric.dist_csr(
|
||||
x1_data=&self.X_data[0],
|
||||
x1_indices=&self.X_indices[0],
|
||||
# Increment the data pointer such that x2_start=0 is aligned with the
|
||||
# j-th row
|
||||
x2_data=&self.Y_data[0] + j * self.n_features,
|
||||
x2_indices=&self.Y_indices[0],
|
||||
x1_start=self.X_indptr[i],
|
||||
x1_end=self.X_indptr[i + 1],
|
||||
x2_start=0,
|
||||
x2_end=self.n_features,
|
||||
size=self.n_features,
|
||||
)
|
||||
|
||||
|
||||
@final
|
||||
cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
|
||||
"""Compute distances between vectors of a dense array and a CSR matrix.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X: ndarray of shape (n_samples_X, n_features)
|
||||
Rows represent vectors. Must be C-contiguous.
|
||||
|
||||
Y: sparse matrix of shape (n_samples_Y, n_features)
|
||||
Rows represent vectors. Must be in CSR format.
|
||||
|
||||
distance_metric: DistanceMetric
|
||||
The distance metric responsible for computing distances
|
||||
between two vectors of (X, Y).
|
||||
"""
|
||||
|
||||
def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
|
||||
super().__init__(distance_metric, n_features=X.shape[1])
|
||||
# Swapping arguments on the constructor
|
||||
self.datasets_pair = SparseDenseDatasetsPair{{name_suffix}}(Y, X, distance_metric)
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_X(self) noexcept nogil:
|
||||
# Swapping interface
|
||||
return self.datasets_pair.n_samples_Y()
|
||||
|
||||
@final
|
||||
cdef intp_t n_samples_Y(self) noexcept nogil:
|
||||
# Swapping interface
|
||||
return self.datasets_pair.n_samples_X()
|
||||
|
||||
@final
|
||||
cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# Swapping arguments on the same interface
|
||||
return self.datasets_pair.surrogate_dist(j, i)
|
||||
|
||||
@final
|
||||
cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
|
||||
# Swapping arguments on the same interface
|
||||
return self.datasets_pair.dist(j, i)
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+228
-228
@@ -1,228 +1,228 @@
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
#
|
||||
# We also use the float64 dtype and C-type names as defined in
|
||||
# `sklearn.utils._typedefs` to maintain consistency.
|
||||
#
|
||||
('64', False, 'float64_t', 'np.float64'),
|
||||
('32', True, 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
from libcpp.vector cimport vector
|
||||
|
||||
from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
|
||||
cdef void _middle_term_sparse_sparse_64(
|
||||
const float64_t[:] X_data,
|
||||
const int32_t[:] X_indices,
|
||||
const int32_t[:] X_indptr,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
const float64_t[:] Y_data,
|
||||
const int32_t[:] Y_indices,
|
||||
const int32_t[:] Y_indptr,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
float64_t * D,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
{{for name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
|
||||
cdef class MiddleTermComputer{{name_suffix}}:
|
||||
cdef:
|
||||
intp_t effective_n_threads
|
||||
intp_t chunks_n_threads
|
||||
intp_t dist_middle_terms_chunks_size
|
||||
intp_t n_features
|
||||
intp_t chunk_size
|
||||
|
||||
# Buffers for the `-2 * X_c @ Y_c.T` term computed via GEMM
|
||||
vector[vector[float64_t]] dist_middle_terms_chunks
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_init(self, intp_t thread_num) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_init(self) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class DenseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
{{if upcast_to_float64}}
|
||||
# Buffers for upcasting chunks of X and Y from 32bit to 64bit
|
||||
vector[vector[float64_t]] X_c_upcast
|
||||
vector[vector[float64_t]] Y_c_upcast
|
||||
{{endif}}
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class SparseSparseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const float64_t[:] X_data
|
||||
const int32_t[:] X_indices
|
||||
const int32_t[:] X_indptr
|
||||
|
||||
const float64_t[:] Y_data
|
||||
const int32_t[:] Y_indices
|
||||
const int32_t[:] Y_indptr
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class SparseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const float64_t[:] X_data
|
||||
const int32_t[:] X_indices
|
||||
const int32_t[:] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
# We treat the dense-sparse case with the sparse-dense case by simply
|
||||
# treating the dist_middle_terms as F-ordered and by swapping arguments.
|
||||
# This attribute is meant to encode the case and adapt the logic
|
||||
# accordingly.
|
||||
bint c_ordered_middle_term
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
{{endfor}}
|
||||
{{py:
|
||||
|
||||
implementation_specific_values = [
|
||||
# Values are the following ones:
|
||||
#
|
||||
# name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE
|
||||
#
|
||||
# We also use the float64 dtype and C-type names as defined in
|
||||
# `sklearn.utils._typedefs` to maintain consistency.
|
||||
#
|
||||
('64', False, 'float64_t', 'np.float64'),
|
||||
('32', True, 'float32_t', 'np.float32')
|
||||
]
|
||||
|
||||
}}
|
||||
from libcpp.vector cimport vector
|
||||
|
||||
from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
|
||||
|
||||
|
||||
cdef void _middle_term_sparse_sparse_64(
|
||||
const float64_t[:] X_data,
|
||||
const int32_t[:] X_indices,
|
||||
const int32_t[:] X_indptr,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
const float64_t[:] Y_data,
|
||||
const int32_t[:] Y_indices,
|
||||
const int32_t[:] Y_indptr,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
float64_t * D,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
{{for name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
|
||||
|
||||
|
||||
cdef class MiddleTermComputer{{name_suffix}}:
|
||||
cdef:
|
||||
intp_t effective_n_threads
|
||||
intp_t chunks_n_threads
|
||||
intp_t dist_middle_terms_chunks_size
|
||||
intp_t n_features
|
||||
intp_t chunk_size
|
||||
|
||||
# Buffers for the `-2 * X_c @ Y_c.T` term computed via GEMM
|
||||
vector[vector[float64_t]] dist_middle_terms_chunks
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_parallel_init(self, intp_t thread_num) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_init(self) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class DenseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] X
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
{{if upcast_to_float64}}
|
||||
# Buffers for upcasting chunks of X and Y from 32bit to 64bit
|
||||
vector[vector[float64_t]] X_c_upcast
|
||||
vector[vector[float64_t]] Y_c_upcast
|
||||
{{endif}}
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_X_init_chunk(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_parallel_init(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class SparseSparseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const float64_t[:] X_data
|
||||
const int32_t[:] X_indices
|
||||
const int32_t[:] X_indptr
|
||||
|
||||
const float64_t[:] Y_data
|
||||
const int32_t[:] Y_indices
|
||||
const int32_t[:] Y_indptr
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class SparseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
|
||||
cdef:
|
||||
const float64_t[:] X_data
|
||||
const int32_t[:] X_indices
|
||||
const int32_t[:] X_indptr
|
||||
|
||||
const {{INPUT_DTYPE_t}}[:, ::1] Y
|
||||
|
||||
# We treat the dense-sparse case with the sparse-dense case by simply
|
||||
# treating the dist_middle_terms as F-ordered and by swapping arguments.
|
||||
# This attribute is meant to encode the case and adapt the logic
|
||||
# accordingly.
|
||||
bint c_ordered_middle_term
|
||||
|
||||
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num
|
||||
) noexcept nogil
|
||||
|
||||
cdef float64_t * _compute_dist_middle_terms(
|
||||
self,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
intp_t Y_start,
|
||||
intp_t Y_end,
|
||||
intp_t thread_num,
|
||||
) noexcept nogil
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+633
-633
File diff suppressed because it is too large
Load Diff
+90
-90
@@ -1,90 +1,90 @@
|
||||
cimport numpy as cnp
|
||||
|
||||
from libcpp.memory cimport shared_ptr
|
||||
from libcpp.vector cimport vector
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
cnp.import_array()
|
||||
|
||||
######################
|
||||
## std::vector to np.ndarray coercion
|
||||
# As type covariance is not supported for C++ containers via Cython,
|
||||
# we need to redefine fused types.
|
||||
ctypedef fused vector_double_intp_t:
|
||||
vector[intp_t]
|
||||
vector[float64_t]
|
||||
|
||||
|
||||
ctypedef fused vector_vector_double_intp_t:
|
||||
vector[vector[intp_t]]
|
||||
vector[vector[float64_t]]
|
||||
|
||||
cdef cnp.ndarray[object, ndim=1] coerce_vectors_to_nd_arrays(
|
||||
shared_ptr[vector_vector_double_intp_t] vecs
|
||||
)
|
||||
|
||||
#####################
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._base cimport BaseDistancesReduction{{name_suffix}}
|
||||
from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
|
||||
|
||||
cdef class RadiusNeighbors{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
|
||||
"""float{{name_suffix}} implementation of the RadiusNeighbors."""
|
||||
|
||||
cdef:
|
||||
float64_t radius
|
||||
|
||||
# DistanceMetric{{name_suffix}} compute rank-preserving surrogate distance via rdist
|
||||
# which are proxies necessitating less computations.
|
||||
# We get the equivalent for the radius to be able to compare it against
|
||||
# vectors' rank-preserving surrogate distances.
|
||||
float64_t r_radius
|
||||
|
||||
# Neighbors indices and distances are returned as np.ndarrays of np.ndarrays.
|
||||
#
|
||||
# For this implementation, we want resizable buffers which we will wrap
|
||||
# into numpy arrays at the end. std::vector comes as a handy container
|
||||
# for interacting efficiently with resizable buffers.
|
||||
#
|
||||
# Though it is possible to access their buffer address with
|
||||
# std::vector::data, they can't be stolen: buffers lifetime
|
||||
# is tied to their std::vector and are deallocated when
|
||||
# std::vectors are.
|
||||
#
|
||||
# To solve this, we dynamically allocate std::vectors and then
|
||||
# encapsulate them in a StdVectorSentinel responsible for
|
||||
# freeing them when the associated np.ndarray is freed.
|
||||
#
|
||||
# Shared pointers (defined via shared_ptr) are use for safer memory management.
|
||||
# Unique pointers (defined via unique_ptr) can't be used as datastructures
|
||||
# are shared across threads for parallel_on_X; see _parallel_on_X_init_chunk.
|
||||
shared_ptr[vector[vector[intp_t]]] neigh_indices
|
||||
shared_ptr[vector[vector[float64_t]]] neigh_distances
|
||||
|
||||
# Used as array of pointers to private datastructures used in threads.
|
||||
vector[shared_ptr[vector[vector[intp_t]]]] neigh_indices_chunks
|
||||
vector[shared_ptr[vector[vector[float64_t]]]] neigh_distances_chunks
|
||||
|
||||
bint sort_results
|
||||
|
||||
@final
|
||||
cdef void _merge_vectors(
|
||||
self,
|
||||
intp_t idx,
|
||||
intp_t num_threads,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class EuclideanRadiusNeighbors{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
|
||||
"""EuclideanDistance-specialisation of RadiusNeighbors{{name_suffix}}."""
|
||||
cdef:
|
||||
MiddleTermComputer{{name_suffix}} middle_term_computer
|
||||
const float64_t[::1] X_norm_squared
|
||||
const float64_t[::1] Y_norm_squared
|
||||
|
||||
bint use_squared_distances
|
||||
|
||||
{{endfor}}
|
||||
cimport numpy as cnp
|
||||
|
||||
from libcpp.memory cimport shared_ptr
|
||||
from libcpp.vector cimport vector
|
||||
from cython cimport final
|
||||
|
||||
from ...utils._typedefs cimport intp_t, float64_t
|
||||
|
||||
cnp.import_array()
|
||||
|
||||
######################
|
||||
## std::vector to np.ndarray coercion
|
||||
# As type covariance is not supported for C++ containers via Cython,
|
||||
# we need to redefine fused types.
|
||||
ctypedef fused vector_double_intp_t:
|
||||
vector[intp_t]
|
||||
vector[float64_t]
|
||||
|
||||
|
||||
ctypedef fused vector_vector_double_intp_t:
|
||||
vector[vector[intp_t]]
|
||||
vector[vector[float64_t]]
|
||||
|
||||
cdef cnp.ndarray[object, ndim=1] coerce_vectors_to_nd_arrays(
|
||||
shared_ptr[vector_vector_double_intp_t] vecs
|
||||
)
|
||||
|
||||
#####################
|
||||
{{for name_suffix in ['64', '32']}}
|
||||
|
||||
from ._base cimport BaseDistancesReduction{{name_suffix}}
|
||||
from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
|
||||
|
||||
cdef class RadiusNeighbors{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
|
||||
"""float{{name_suffix}} implementation of the RadiusNeighbors."""
|
||||
|
||||
cdef:
|
||||
float64_t radius
|
||||
|
||||
# DistanceMetric{{name_suffix}} compute rank-preserving surrogate distance via rdist
|
||||
# which are proxies necessitating less computations.
|
||||
# We get the equivalent for the radius to be able to compare it against
|
||||
# vectors' rank-preserving surrogate distances.
|
||||
float64_t r_radius
|
||||
|
||||
# Neighbors indices and distances are returned as np.ndarrays of np.ndarrays.
|
||||
#
|
||||
# For this implementation, we want resizable buffers which we will wrap
|
||||
# into numpy arrays at the end. std::vector comes as a handy container
|
||||
# for interacting efficiently with resizable buffers.
|
||||
#
|
||||
# Though it is possible to access their buffer address with
|
||||
# std::vector::data, they can't be stolen: buffers lifetime
|
||||
# is tied to their std::vector and are deallocated when
|
||||
# std::vectors are.
|
||||
#
|
||||
# To solve this, we dynamically allocate std::vectors and then
|
||||
# encapsulate them in a StdVectorSentinel responsible for
|
||||
# freeing them when the associated np.ndarray is freed.
|
||||
#
|
||||
# Shared pointers (defined via shared_ptr) are use for safer memory management.
|
||||
# Unique pointers (defined via unique_ptr) can't be used as datastructures
|
||||
# are shared across threads for parallel_on_X; see _parallel_on_X_init_chunk.
|
||||
shared_ptr[vector[vector[intp_t]]] neigh_indices
|
||||
shared_ptr[vector[vector[float64_t]]] neigh_distances
|
||||
|
||||
# Used as array of pointers to private datastructures used in threads.
|
||||
vector[shared_ptr[vector[vector[intp_t]]]] neigh_indices_chunks
|
||||
vector[shared_ptr[vector[vector[float64_t]]]] neigh_distances_chunks
|
||||
|
||||
bint sort_results
|
||||
|
||||
@final
|
||||
cdef void _merge_vectors(
|
||||
self,
|
||||
intp_t idx,
|
||||
intp_t num_threads,
|
||||
) noexcept nogil
|
||||
|
||||
|
||||
cdef class EuclideanRadiusNeighbors{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
|
||||
"""EuclideanDistance-specialisation of RadiusNeighbors{{name_suffix}}."""
|
||||
cdef:
|
||||
MiddleTermComputer{{name_suffix}} middle_term_computer
|
||||
const float64_t[::1] X_norm_squared
|
||||
const float64_t[::1] Y_norm_squared
|
||||
|
||||
bint use_squared_distances
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+514
-514
File diff suppressed because it is too large
Load Diff
+217
-217
@@ -1,217 +1,217 @@
|
||||
import warnings
|
||||
|
||||
from cython cimport floating, final, integral
|
||||
from cython.operator cimport dereference as deref
|
||||
from cython.parallel cimport parallel, prange
|
||||
from ._classmode cimport WeightingStrategy
|
||||
from ...utils._typedefs cimport intp_t, float64_t, uint8_t
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import issparse
|
||||
from ...utils.parallel import _get_threadpool_controller
|
||||
|
||||
|
||||
{{for name_suffix in ["32", "64"]}}
|
||||
from ._radius_neighbors cimport RadiusNeighbors{{name_suffix}}
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
cdef class RadiusNeighborsClassMode{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
|
||||
"""
|
||||
{{name_suffix}}bit implementation of RadiusNeighborsClassMode.
|
||||
"""
|
||||
cdef:
|
||||
const intp_t[::1] Y_labels
|
||||
const intp_t[::1] unique_Y_labels
|
||||
intp_t outlier_label_index
|
||||
bint outlier_label_exists
|
||||
bint outliers_exist
|
||||
uint8_t[::1] outliers
|
||||
object outlier_label
|
||||
float64_t[:, ::1] class_scores
|
||||
WeightingStrategy weight_type
|
||||
|
||||
@classmethod
|
||||
def compute(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
float64_t radius,
|
||||
weights,
|
||||
Y_labels,
|
||||
unique_Y_labels,
|
||||
outlier_label=None,
|
||||
str metric="euclidean",
|
||||
chunk_size=None,
|
||||
dict metric_kwargs=None,
|
||||
str strategy=None,
|
||||
):
|
||||
# Use a generic implementation that handles most scipy
|
||||
# metrics by computing the distances between 2 vectors at a time.
|
||||
pda = RadiusNeighborsClassMode{{name_suffix}}(
|
||||
datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
|
||||
radius=radius,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
weights=weights,
|
||||
Y_labels=Y_labels,
|
||||
unique_Y_labels=unique_Y_labels,
|
||||
outlier_label=outlier_label,
|
||||
)
|
||||
|
||||
# Limit the number of threads in second level of nested parallelism for BLAS
|
||||
# to avoid threads over-subscription (in GEMM for instance).
|
||||
with _get_threadpool_controller().limit(limits=1, user_api="blas"):
|
||||
if pda.execute_in_parallel_on_Y:
|
||||
pda._parallel_on_Y()
|
||||
else:
|
||||
pda._parallel_on_X()
|
||||
|
||||
return pda._finalize_results()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
DatasetsPair{{name_suffix}} datasets_pair,
|
||||
const intp_t[::1] Y_labels,
|
||||
const intp_t[::1] unique_Y_labels,
|
||||
float64_t radius,
|
||||
chunk_size=None,
|
||||
strategy=None,
|
||||
weights=None,
|
||||
outlier_label=None,
|
||||
):
|
||||
super().__init__(
|
||||
datasets_pair=datasets_pair,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
radius=radius,
|
||||
)
|
||||
|
||||
if weights == "uniform":
|
||||
self.weight_type = WeightingStrategy.uniform
|
||||
elif weights == "distance":
|
||||
self.weight_type = WeightingStrategy.distance
|
||||
else:
|
||||
self.weight_type = WeightingStrategy.callable
|
||||
|
||||
self.Y_labels = Y_labels
|
||||
self.unique_Y_labels = unique_Y_labels
|
||||
self.outlier_label_index = -1
|
||||
self.outliers_exist = False
|
||||
self.outlier_label = outlier_label
|
||||
self.outliers = np.zeros(self.n_samples_X, dtype=np.bool_)
|
||||
|
||||
cdef intp_t idx
|
||||
if self.outlier_label is not None:
|
||||
for idx in range(self.unique_Y_labels.shape[0]):
|
||||
if self.unique_Y_labels[idx] == outlier_label:
|
||||
self.outlier_label_index = idx
|
||||
|
||||
# Map from set of unique labels to their indices in `class_scores`
|
||||
# Buffer used in building a histogram for one-pass weighted mode
|
||||
self.class_scores = np.zeros(
|
||||
(self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
|
||||
)
|
||||
|
||||
|
||||
cdef inline void weighted_histogram_mode(
|
||||
self,
|
||||
intp_t sample_index,
|
||||
intp_t sample_n_neighbors,
|
||||
intp_t* indices,
|
||||
float64_t* distances,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t neighbor_idx, neighbor_class_idx, label_index
|
||||
float64_t score_incr = 1
|
||||
bint use_distance_weighting = (
|
||||
self.weight_type == WeightingStrategy.distance
|
||||
)
|
||||
|
||||
if sample_n_neighbors == 0:
|
||||
self.outliers_exist = True
|
||||
self.outliers[sample_index] = True
|
||||
if self.outlier_label_index >= 0:
|
||||
self.class_scores[sample_index][self.outlier_label_index] = score_incr
|
||||
|
||||
return
|
||||
|
||||
# Iterate over the neighbors. This can be different for
|
||||
# each of the samples as they are based on the radius.
|
||||
for neighbor_rank in range(sample_n_neighbors):
|
||||
if use_distance_weighting:
|
||||
score_incr = 1 / distances[neighbor_rank]
|
||||
|
||||
neighbor_idx = indices[neighbor_rank]
|
||||
neighbor_class_idx = self.Y_labels[neighbor_idx]
|
||||
self.class_scores[sample_index][neighbor_class_idx] += score_incr
|
||||
|
||||
return
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx
|
||||
|
||||
for idx in range(X_start, X_end):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index=idx,
|
||||
sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
|
||||
indices=deref(self.neigh_indices)[idx].data(),
|
||||
distances=deref(self.neigh_distances)[idx].data(),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx
|
||||
|
||||
with nogil, parallel(num_threads=self.effective_n_threads):
|
||||
# Merge vectors used in threads into the main ones.
|
||||
# This is done in parallel sample-wise (no need for locks).
|
||||
for idx in prange(self.n_samples_X, schedule='static'):
|
||||
self._merge_vectors(idx, self.chunks_n_threads)
|
||||
|
||||
for idx in prange(self.n_samples_X, schedule='static'):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index=idx,
|
||||
sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
|
||||
indices=deref(self.neigh_indices)[idx].data(),
|
||||
distances=deref(self.neigh_distances)[idx].data(),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
def _finalize_results(self):
|
||||
if self.outliers_exist and self.outlier_label is None:
|
||||
raise ValueError(
|
||||
"No neighbors found for test samples %r, "
|
||||
"you can try using larger radius, "
|
||||
"giving a label for outliers, "
|
||||
"or considering removing them from your dataset."
|
||||
% np.where(self.outliers)[0]
|
||||
)
|
||||
|
||||
if self.outliers_exist and self.outlier_label_index < 0:
|
||||
warnings.warn(
|
||||
"Outlier label %s is not in training "
|
||||
"classes. All class probabilities of "
|
||||
"outliers will be assigned with 0."
|
||||
% self.outlier_label
|
||||
)
|
||||
|
||||
probabilities = np.asarray(self.class_scores)
|
||||
normalizer = probabilities.sum(axis=1, keepdims=True)
|
||||
normalizer[normalizer == 0.0] = 1.0
|
||||
probabilities /= normalizer
|
||||
return probabilities
|
||||
|
||||
{{endfor}}
|
||||
import warnings
|
||||
|
||||
from cython cimport floating, final, integral
|
||||
from cython.operator cimport dereference as deref
|
||||
from cython.parallel cimport parallel, prange
|
||||
from ._classmode cimport WeightingStrategy
|
||||
from ...utils._typedefs cimport intp_t, float64_t, uint8_t
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import issparse
|
||||
from ...utils.parallel import _get_threadpool_controller
|
||||
|
||||
|
||||
{{for name_suffix in ["32", "64"]}}
|
||||
from ._radius_neighbors cimport RadiusNeighbors{{name_suffix}}
|
||||
from ._datasets_pair cimport DatasetsPair{{name_suffix}}
|
||||
|
||||
cdef class RadiusNeighborsClassMode{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
|
||||
"""
|
||||
{{name_suffix}}bit implementation of RadiusNeighborsClassMode.
|
||||
"""
|
||||
cdef:
|
||||
const intp_t[::1] Y_labels
|
||||
const intp_t[::1] unique_Y_labels
|
||||
intp_t outlier_label_index
|
||||
bint outlier_label_exists
|
||||
bint outliers_exist
|
||||
uint8_t[::1] outliers
|
||||
object outlier_label
|
||||
float64_t[:, ::1] class_scores
|
||||
WeightingStrategy weight_type
|
||||
|
||||
@classmethod
|
||||
def compute(
|
||||
cls,
|
||||
X,
|
||||
Y,
|
||||
float64_t radius,
|
||||
weights,
|
||||
Y_labels,
|
||||
unique_Y_labels,
|
||||
outlier_label=None,
|
||||
str metric="euclidean",
|
||||
chunk_size=None,
|
||||
dict metric_kwargs=None,
|
||||
str strategy=None,
|
||||
):
|
||||
# Use a generic implementation that handles most scipy
|
||||
# metrics by computing the distances between 2 vectors at a time.
|
||||
pda = RadiusNeighborsClassMode{{name_suffix}}(
|
||||
datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
|
||||
radius=radius,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
weights=weights,
|
||||
Y_labels=Y_labels,
|
||||
unique_Y_labels=unique_Y_labels,
|
||||
outlier_label=outlier_label,
|
||||
)
|
||||
|
||||
# Limit the number of threads in second level of nested parallelism for BLAS
|
||||
# to avoid threads over-subscription (in GEMM for instance).
|
||||
with _get_threadpool_controller().limit(limits=1, user_api="blas"):
|
||||
if pda.execute_in_parallel_on_Y:
|
||||
pda._parallel_on_Y()
|
||||
else:
|
||||
pda._parallel_on_X()
|
||||
|
||||
return pda._finalize_results()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
DatasetsPair{{name_suffix}} datasets_pair,
|
||||
const intp_t[::1] Y_labels,
|
||||
const intp_t[::1] unique_Y_labels,
|
||||
float64_t radius,
|
||||
chunk_size=None,
|
||||
strategy=None,
|
||||
weights=None,
|
||||
outlier_label=None,
|
||||
):
|
||||
super().__init__(
|
||||
datasets_pair=datasets_pair,
|
||||
chunk_size=chunk_size,
|
||||
strategy=strategy,
|
||||
radius=radius,
|
||||
)
|
||||
|
||||
if weights == "uniform":
|
||||
self.weight_type = WeightingStrategy.uniform
|
||||
elif weights == "distance":
|
||||
self.weight_type = WeightingStrategy.distance
|
||||
else:
|
||||
self.weight_type = WeightingStrategy.callable
|
||||
|
||||
self.Y_labels = Y_labels
|
||||
self.unique_Y_labels = unique_Y_labels
|
||||
self.outlier_label_index = -1
|
||||
self.outliers_exist = False
|
||||
self.outlier_label = outlier_label
|
||||
self.outliers = np.zeros(self.n_samples_X, dtype=np.bool_)
|
||||
|
||||
cdef intp_t idx
|
||||
if self.outlier_label is not None:
|
||||
for idx in range(self.unique_Y_labels.shape[0]):
|
||||
if self.unique_Y_labels[idx] == outlier_label:
|
||||
self.outlier_label_index = idx
|
||||
|
||||
# Map from set of unique labels to their indices in `class_scores`
|
||||
# Buffer used in building a histogram for one-pass weighted mode
|
||||
self.class_scores = np.zeros(
|
||||
(self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
|
||||
)
|
||||
|
||||
|
||||
cdef inline void weighted_histogram_mode(
|
||||
self,
|
||||
intp_t sample_index,
|
||||
intp_t sample_n_neighbors,
|
||||
intp_t* indices,
|
||||
float64_t* distances,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t neighbor_idx, neighbor_class_idx, label_index
|
||||
float64_t score_incr = 1
|
||||
bint use_distance_weighting = (
|
||||
self.weight_type == WeightingStrategy.distance
|
||||
)
|
||||
|
||||
if sample_n_neighbors == 0:
|
||||
self.outliers_exist = True
|
||||
self.outliers[sample_index] = True
|
||||
if self.outlier_label_index >= 0:
|
||||
self.class_scores[sample_index][self.outlier_label_index] = score_incr
|
||||
|
||||
return
|
||||
|
||||
# Iterate over the neighbors. This can be different for
|
||||
# each of the samples as they are based on the radius.
|
||||
for neighbor_rank in range(sample_n_neighbors):
|
||||
if use_distance_weighting:
|
||||
score_incr = 1 / distances[neighbor_rank]
|
||||
|
||||
neighbor_idx = indices[neighbor_rank]
|
||||
neighbor_class_idx = self.Y_labels[neighbor_idx]
|
||||
self.class_scores[sample_index][neighbor_class_idx] += score_incr
|
||||
|
||||
return
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_X_prange_iter_finalize(
|
||||
self,
|
||||
intp_t thread_num,
|
||||
intp_t X_start,
|
||||
intp_t X_end,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx
|
||||
|
||||
for idx in range(X_start, X_end):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index=idx,
|
||||
sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
|
||||
indices=deref(self.neigh_indices)[idx].data(),
|
||||
distances=deref(self.neigh_distances)[idx].data(),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
@final
|
||||
cdef void _parallel_on_Y_finalize(
|
||||
self,
|
||||
) noexcept nogil:
|
||||
cdef:
|
||||
intp_t idx
|
||||
|
||||
with nogil, parallel(num_threads=self.effective_n_threads):
|
||||
# Merge vectors used in threads into the main ones.
|
||||
# This is done in parallel sample-wise (no need for locks).
|
||||
for idx in prange(self.n_samples_X, schedule='static'):
|
||||
self._merge_vectors(idx, self.chunks_n_threads)
|
||||
|
||||
for idx in prange(self.n_samples_X, schedule='static'):
|
||||
self.weighted_histogram_mode(
|
||||
sample_index=idx,
|
||||
sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
|
||||
indices=deref(self.neigh_indices)[idx].data(),
|
||||
distances=deref(self.neigh_distances)[idx].data(),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
def _finalize_results(self):
|
||||
if self.outliers_exist and self.outlier_label is None:
|
||||
raise ValueError(
|
||||
"No neighbors found for test samples %r, "
|
||||
"you can try using larger radius, "
|
||||
"giving a label for outliers, "
|
||||
"or considering removing them from your dataset."
|
||||
% np.where(self.outliers)[0]
|
||||
)
|
||||
|
||||
if self.outliers_exist and self.outlier_label_index < 0:
|
||||
warnings.warn(
|
||||
"Outlier label %s is not in training "
|
||||
"classes. All class probabilities of "
|
||||
"outliers will be assigned with 0."
|
||||
% self.outlier_label
|
||||
)
|
||||
|
||||
probabilities = np.asarray(self.class_scores)
|
||||
normalizer = probabilities.sum(axis=1, keepdims=True)
|
||||
normalizer[normalizer == 0.0] = 1.0
|
||||
probabilities /= normalizer
|
||||
return probabilities
|
||||
|
||||
{{endfor}}
|
||||
|
||||
+193
-193
@@ -1,193 +1,193 @@
|
||||
# Note: the dependencies between different Cython files in
|
||||
# _pairwise_distances_reduction is probably one of the most involved in
|
||||
# scikit-learn. If you change this file make sure you build from scratch:
|
||||
# rm -rf build; make dev-meson
|
||||
# run a command like this:
|
||||
# ninja -C build/cp312 -t missingdeps
|
||||
# and make sure that the output is something like:
|
||||
# No missing dependencies on generated files found.
|
||||
|
||||
# _pairwise_distances_reduction is cimported from other subpackages so this is
|
||||
# needed for the cimport to work
|
||||
_pairwise_distances_reduction_cython_tree = [
|
||||
fs.copyfile('__init__.py'),
|
||||
# We are in a sub-module of metrics, so we always need to have
|
||||
# sklearn/metrics/__init__.py copied to the build directory to avoid the
|
||||
# error:
|
||||
# relative cimport beyond main package is not allowed
|
||||
metrics_cython_tree
|
||||
]
|
||||
|
||||
_classmode_pxd = fs.copyfile('_classmode.pxd')
|
||||
|
||||
_datasets_pair_pxd = custom_target(
|
||||
'_datasets_pair_pxd',
|
||||
output: '_datasets_pair.pxd',
|
||||
input: '_datasets_pair.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_datasets_pair_pyx = custom_target(
|
||||
'_datasets_pair_pyx',
|
||||
output: '_datasets_pair.pyx',
|
||||
input: '_datasets_pair.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_datasets_pair_pxd, _pairwise_distances_reduction_cython_tree, utils_cython_tree],
|
||||
)
|
||||
_datasets_pair = py.extension_module(
|
||||
'_datasets_pair',
|
||||
cython_gen_cpp.process(_datasets_pair_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_base_pxd = custom_target(
|
||||
'_base_pxd',
|
||||
output: '_base.pxd',
|
||||
input: '_base.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_base_pyx = custom_target(
|
||||
'_base_pyx',
|
||||
output: '_base.pyx',
|
||||
input: '_base.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_base_pxd, _pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, utils_cython_tree],
|
||||
)
|
||||
_base = py.extension_module(
|
||||
'_base',
|
||||
cython_gen_cpp.process(_base_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_middle_term_computer_pxd = custom_target(
|
||||
'_middle_term_computer_pxd',
|
||||
output: '_middle_term_computer.pxd',
|
||||
input: '_middle_term_computer.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_middle_term_computer_pyx = custom_target(
|
||||
'_middle_term_computer_pyx',
|
||||
output: '_middle_term_computer.pyx',
|
||||
input: '_middle_term_computer.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_middle_term_computer_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
utils_cython_tree],
|
||||
)
|
||||
_middle_term_computer = py.extension_module(
|
||||
'_middle_term_computer',
|
||||
cython_gen_cpp.process(_middle_term_computer_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_argkmin_pxd = custom_target(
|
||||
'_argkmin_pxd',
|
||||
output: '_argkmin.pxd',
|
||||
input: '_argkmin.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_argkmin_pyx = custom_target(
|
||||
'_argkmin_pyx',
|
||||
output: '_argkmin.pyx',
|
||||
input: '_argkmin.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_argkmin_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd],
|
||||
)
|
||||
_argkmin = py.extension_module(
|
||||
'_argkmin',
|
||||
cython_gen_cpp.process(_argkmin_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_radius_neighbors_pxd = custom_target(
|
||||
'_radius_neighbors_pxd',
|
||||
output: '_radius_neighbors.pxd',
|
||||
input: '_radius_neighbors.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_radius_neighbors_pyx = custom_target(
|
||||
'_radius_neighbors_pyx',
|
||||
output: '_radius_neighbors.pyx',
|
||||
input: '_radius_neighbors.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_radius_neighbors_pxd,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd,
|
||||
_pairwise_distances_reduction_cython_tree, utils_cython_tree],
|
||||
)
|
||||
_radius_neighbors = py.extension_module(
|
||||
'_radius_neighbors',
|
||||
cython_gen_cpp.process(_radius_neighbors_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_argkmin_classmode_pyx = custom_target(
|
||||
'_argkmin_classmode_pyx',
|
||||
output: '_argkmin_classmode.pyx',
|
||||
input: '_argkmin_classmode.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_classmode_pxd,
|
||||
_argkmin_pxd, _pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd, utils_cython_tree],
|
||||
)
|
||||
_argkmin_classmode = py.extension_module(
|
||||
'_argkmin_classmode',
|
||||
cython_gen_cpp.process(_argkmin_classmode_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
# XXX: for some reason -fno-sized-deallocation is needed otherwise there is
|
||||
# an error with undefined symbol _ZdlPv at import time in manylinux wheels.
|
||||
# See https://github.com/scikit-learn/scikit-learn/issues/28596 for more details.
|
||||
cpp_args: ['-fno-sized-deallocation'],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_radius_neighbors_classmode_pyx = custom_target(
|
||||
'_radius_neighbors_classmode_pyx',
|
||||
output: '_radius_neighbors_classmode.pyx',
|
||||
input: '_radius_neighbors_classmode.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_classmode_pxd,
|
||||
_middle_term_computer_pxd, _radius_neighbors_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, utils_cython_tree],
|
||||
)
|
||||
_radius_neighbors_classmode = py.extension_module(
|
||||
'_radius_neighbors_classmode',
|
||||
cython_gen_cpp.process(_radius_neighbors_classmode_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
# Note: the dependencies between different Cython files in
|
||||
# _pairwise_distances_reduction is probably one of the most involved in
|
||||
# scikit-learn. If you change this file make sure you build from scratch:
|
||||
# rm -rf build; make dev-meson
|
||||
# run a command like this:
|
||||
# ninja -C build/cp312 -t missingdeps
|
||||
# and make sure that the output is something like:
|
||||
# No missing dependencies on generated files found.
|
||||
|
||||
# _pairwise_distances_reduction is cimported from other subpackages so this is
|
||||
# needed for the cimport to work
|
||||
_pairwise_distances_reduction_cython_tree = [
|
||||
fs.copyfile('__init__.py'),
|
||||
# We are in a sub-module of metrics, so we always need to have
|
||||
# sklearn/metrics/__init__.py copied to the build directory to avoid the
|
||||
# error:
|
||||
# relative cimport beyond main package is not allowed
|
||||
metrics_cython_tree
|
||||
]
|
||||
|
||||
_classmode_pxd = fs.copyfile('_classmode.pxd')
|
||||
|
||||
_datasets_pair_pxd = custom_target(
|
||||
'_datasets_pair_pxd',
|
||||
output: '_datasets_pair.pxd',
|
||||
input: '_datasets_pair.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_datasets_pair_pyx = custom_target(
|
||||
'_datasets_pair_pyx',
|
||||
output: '_datasets_pair.pyx',
|
||||
input: '_datasets_pair.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_datasets_pair_pxd, _pairwise_distances_reduction_cython_tree, utils_cython_tree],
|
||||
)
|
||||
_datasets_pair = py.extension_module(
|
||||
'_datasets_pair',
|
||||
cython_gen_cpp.process(_datasets_pair_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_base_pxd = custom_target(
|
||||
'_base_pxd',
|
||||
output: '_base.pxd',
|
||||
input: '_base.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_base_pyx = custom_target(
|
||||
'_base_pyx',
|
||||
output: '_base.pyx',
|
||||
input: '_base.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_base_pxd, _pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, utils_cython_tree],
|
||||
)
|
||||
_base = py.extension_module(
|
||||
'_base',
|
||||
cython_gen_cpp.process(_base_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_middle_term_computer_pxd = custom_target(
|
||||
'_middle_term_computer_pxd',
|
||||
output: '_middle_term_computer.pxd',
|
||||
input: '_middle_term_computer.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_middle_term_computer_pyx = custom_target(
|
||||
'_middle_term_computer_pyx',
|
||||
output: '_middle_term_computer.pyx',
|
||||
input: '_middle_term_computer.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_middle_term_computer_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
utils_cython_tree],
|
||||
)
|
||||
_middle_term_computer = py.extension_module(
|
||||
'_middle_term_computer',
|
||||
cython_gen_cpp.process(_middle_term_computer_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_argkmin_pxd = custom_target(
|
||||
'_argkmin_pxd',
|
||||
output: '_argkmin.pxd',
|
||||
input: '_argkmin.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_argkmin_pyx = custom_target(
|
||||
'_argkmin_pyx',
|
||||
output: '_argkmin.pyx',
|
||||
input: '_argkmin.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_argkmin_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd],
|
||||
)
|
||||
_argkmin = py.extension_module(
|
||||
'_argkmin',
|
||||
cython_gen_cpp.process(_argkmin_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_radius_neighbors_pxd = custom_target(
|
||||
'_radius_neighbors_pxd',
|
||||
output: '_radius_neighbors.pxd',
|
||||
input: '_radius_neighbors.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
|
||||
)
|
||||
_radius_neighbors_pyx = custom_target(
|
||||
'_radius_neighbors_pyx',
|
||||
output: '_radius_neighbors.pyx',
|
||||
input: '_radius_neighbors.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_radius_neighbors_pxd,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd,
|
||||
_pairwise_distances_reduction_cython_tree, utils_cython_tree],
|
||||
)
|
||||
_radius_neighbors = py.extension_module(
|
||||
'_radius_neighbors',
|
||||
cython_gen_cpp.process(_radius_neighbors_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_argkmin_classmode_pyx = custom_target(
|
||||
'_argkmin_classmode_pyx',
|
||||
output: '_argkmin_classmode.pyx',
|
||||
input: '_argkmin_classmode.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_classmode_pxd,
|
||||
_argkmin_pxd, _pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd, utils_cython_tree],
|
||||
)
|
||||
_argkmin_classmode = py.extension_module(
|
||||
'_argkmin_classmode',
|
||||
cython_gen_cpp.process(_argkmin_classmode_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
# XXX: for some reason -fno-sized-deallocation is needed otherwise there is
|
||||
# an error with undefined symbol _ZdlPv at import time in manylinux wheels.
|
||||
# See https://github.com/scikit-learn/scikit-learn/issues/28596 for more details.
|
||||
cpp_args: ['-fno-sized-deallocation'],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
_radius_neighbors_classmode_pyx = custom_target(
|
||||
'_radius_neighbors_classmode_pyx',
|
||||
output: '_radius_neighbors_classmode.pyx',
|
||||
input: '_radius_neighbors_classmode.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: [_classmode_pxd,
|
||||
_middle_term_computer_pxd, _radius_neighbors_pxd,
|
||||
_pairwise_distances_reduction_cython_tree,
|
||||
_datasets_pair_pxd, _base_pxd, utils_cython_tree],
|
||||
)
|
||||
_radius_neighbors_classmode = py.extension_module(
|
||||
'_radius_neighbors_classmode',
|
||||
cython_gen_cpp.process(_radius_neighbors_classmode_pyx),
|
||||
dependencies: [np_dep, openmp_dep],
|
||||
subdir: 'sklearn/metrics/_pairwise_distances_reduction',
|
||||
install: true
|
||||
)
|
||||
|
||||
@@ -1,107 +1,107 @@
|
||||
# Authors: The scikit-learn developers
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
from cython cimport floating
|
||||
from cython.parallel cimport prange
|
||||
from libc.math cimport fabs
|
||||
|
||||
from ..utils._typedefs cimport intp_t
|
||||
|
||||
from ..utils._openmp_helpers import _openmp_effective_n_threads
|
||||
|
||||
|
||||
def _chi2_kernel_fast(floating[:, :] X,
|
||||
floating[:, :] Y,
|
||||
floating[:, :] result):
|
||||
cdef intp_t i, j, k
|
||||
cdef intp_t n_samples_X = X.shape[0]
|
||||
cdef intp_t n_samples_Y = Y.shape[0]
|
||||
cdef intp_t n_features = X.shape[1]
|
||||
cdef double res, nom, denom
|
||||
|
||||
with nogil:
|
||||
for i in range(n_samples_X):
|
||||
for j in range(n_samples_Y):
|
||||
res = 0
|
||||
for k in range(n_features):
|
||||
denom = (X[i, k] - Y[j, k])
|
||||
nom = (X[i, k] + Y[j, k])
|
||||
if nom != 0:
|
||||
res += denom * denom / nom
|
||||
result[i, j] = -res
|
||||
|
||||
|
||||
def _sparse_manhattan(
|
||||
const floating[::1] X_data,
|
||||
const int[:] X_indices,
|
||||
const int[:] X_indptr,
|
||||
const floating[::1] Y_data,
|
||||
const int[:] Y_indices,
|
||||
const int[:] Y_indptr,
|
||||
double[:, ::1] D,
|
||||
):
|
||||
"""Pairwise L1 distances for CSR matrices.
|
||||
|
||||
Usage:
|
||||
>>> D = np.zeros(X.shape[0], Y.shape[0])
|
||||
>>> _sparse_manhattan(X.data, X.indices, X.indptr,
|
||||
... Y.data, Y.indices, Y.indptr,
|
||||
... D)
|
||||
"""
|
||||
cdef intp_t px, py, i, j, ix, iy
|
||||
cdef double d = 0.0
|
||||
|
||||
cdef int m = D.shape[0]
|
||||
cdef int n = D.shape[1]
|
||||
|
||||
cdef int X_indptr_end = 0
|
||||
cdef int Y_indptr_end = 0
|
||||
|
||||
cdef int num_threads = _openmp_effective_n_threads()
|
||||
|
||||
# We scan the matrices row by row.
|
||||
# Given row px in X and row py in Y, we find the positions (i and j
|
||||
# respectively), in .indices where the indices for the two rows start.
|
||||
# If the indices (ix and iy) are the same, the corresponding data values
|
||||
# are processed and the cursors i and j are advanced.
|
||||
# If not, the lowest index is considered. Its associated data value is
|
||||
# processed and its cursor is advanced.
|
||||
# We proceed like this until one of the cursors hits the end for its row.
|
||||
# Then we process all remaining data values in the other row.
|
||||
|
||||
# Below the avoidance of inplace operators is intentional.
|
||||
# When prange is used, the inplace operator has a special meaning, i.e. it
|
||||
# signals a "reduction"
|
||||
|
||||
for px in prange(m, nogil=True, num_threads=num_threads):
|
||||
X_indptr_end = X_indptr[px + 1]
|
||||
for py in range(n):
|
||||
Y_indptr_end = Y_indptr[py + 1]
|
||||
i = X_indptr[px]
|
||||
j = Y_indptr[py]
|
||||
d = 0.0
|
||||
while i < X_indptr_end and j < Y_indptr_end:
|
||||
ix = X_indices[i]
|
||||
iy = Y_indices[j]
|
||||
|
||||
if ix == iy:
|
||||
d = d + fabs(X_data[i] - Y_data[j])
|
||||
i = i + 1
|
||||
j = j + 1
|
||||
elif ix < iy:
|
||||
d = d + fabs(X_data[i])
|
||||
i = i + 1
|
||||
else:
|
||||
d = d + fabs(Y_data[j])
|
||||
j = j + 1
|
||||
|
||||
if i == X_indptr_end:
|
||||
while j < Y_indptr_end:
|
||||
d = d + fabs(Y_data[j])
|
||||
j = j + 1
|
||||
else:
|
||||
while i < X_indptr_end:
|
||||
d = d + fabs(X_data[i])
|
||||
i = i + 1
|
||||
|
||||
D[px, py] = d
|
||||
# Authors: The scikit-learn developers
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
from cython cimport floating
|
||||
from cython.parallel cimport prange
|
||||
from libc.math cimport fabs
|
||||
|
||||
from ..utils._typedefs cimport intp_t
|
||||
|
||||
from ..utils._openmp_helpers import _openmp_effective_n_threads
|
||||
|
||||
|
||||
def _chi2_kernel_fast(floating[:, :] X,
|
||||
floating[:, :] Y,
|
||||
floating[:, :] result):
|
||||
cdef intp_t i, j, k
|
||||
cdef intp_t n_samples_X = X.shape[0]
|
||||
cdef intp_t n_samples_Y = Y.shape[0]
|
||||
cdef intp_t n_features = X.shape[1]
|
||||
cdef double res, nom, denom
|
||||
|
||||
with nogil:
|
||||
for i in range(n_samples_X):
|
||||
for j in range(n_samples_Y):
|
||||
res = 0
|
||||
for k in range(n_features):
|
||||
denom = (X[i, k] - Y[j, k])
|
||||
nom = (X[i, k] + Y[j, k])
|
||||
if nom != 0:
|
||||
res += denom * denom / nom
|
||||
result[i, j] = -res
|
||||
|
||||
|
||||
def _sparse_manhattan(
|
||||
const floating[::1] X_data,
|
||||
const int[:] X_indices,
|
||||
const int[:] X_indptr,
|
||||
const floating[::1] Y_data,
|
||||
const int[:] Y_indices,
|
||||
const int[:] Y_indptr,
|
||||
double[:, ::1] D,
|
||||
):
|
||||
"""Pairwise L1 distances for CSR matrices.
|
||||
|
||||
Usage:
|
||||
>>> D = np.zeros(X.shape[0], Y.shape[0])
|
||||
>>> _sparse_manhattan(X.data, X.indices, X.indptr,
|
||||
... Y.data, Y.indices, Y.indptr,
|
||||
... D)
|
||||
"""
|
||||
cdef intp_t px, py, i, j, ix, iy
|
||||
cdef double d = 0.0
|
||||
|
||||
cdef int m = D.shape[0]
|
||||
cdef int n = D.shape[1]
|
||||
|
||||
cdef int X_indptr_end = 0
|
||||
cdef int Y_indptr_end = 0
|
||||
|
||||
cdef int num_threads = _openmp_effective_n_threads()
|
||||
|
||||
# We scan the matrices row by row.
|
||||
# Given row px in X and row py in Y, we find the positions (i and j
|
||||
# respectively), in .indices where the indices for the two rows start.
|
||||
# If the indices (ix and iy) are the same, the corresponding data values
|
||||
# are processed and the cursors i and j are advanced.
|
||||
# If not, the lowest index is considered. Its associated data value is
|
||||
# processed and its cursor is advanced.
|
||||
# We proceed like this until one of the cursors hits the end for its row.
|
||||
# Then we process all remaining data values in the other row.
|
||||
|
||||
# Below the avoidance of inplace operators is intentional.
|
||||
# When prange is used, the inplace operator has a special meaning, i.e. it
|
||||
# signals a "reduction"
|
||||
|
||||
for px in prange(m, nogil=True, num_threads=num_threads):
|
||||
X_indptr_end = X_indptr[px + 1]
|
||||
for py in range(n):
|
||||
Y_indptr_end = Y_indptr[py + 1]
|
||||
i = X_indptr[px]
|
||||
j = Y_indptr[py]
|
||||
d = 0.0
|
||||
while i < X_indptr_end and j < Y_indptr_end:
|
||||
ix = X_indices[i]
|
||||
iy = Y_indices[j]
|
||||
|
||||
if ix == iy:
|
||||
d = d + fabs(X_data[i] - Y_data[j])
|
||||
i = i + 1
|
||||
j = j + 1
|
||||
elif ix < iy:
|
||||
d = d + fabs(X_data[i])
|
||||
i = i + 1
|
||||
else:
|
||||
d = d + fabs(Y_data[j])
|
||||
j = j + 1
|
||||
|
||||
if i == X_indptr_end:
|
||||
while j < Y_indptr_end:
|
||||
d = d + fabs(Y_data[j])
|
||||
j = j + 1
|
||||
else:
|
||||
while i < X_indptr_end:
|
||||
d = d + fabs(X_data[i])
|
||||
i = i + 1
|
||||
|
||||
D[px, py] = d
|
||||
|
||||
+69
-69
@@ -1,69 +1,69 @@
|
||||
# Authors: The scikit-learn developers
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
from libc.math cimport exp, lgamma
|
||||
|
||||
from ...utils._typedefs cimport float64_t, int64_t
|
||||
|
||||
import numpy as np
|
||||
from scipy.special import gammaln
|
||||
|
||||
|
||||
def expected_mutual_information(contingency, int64_t n_samples):
|
||||
"""Calculate the expected mutual information for two labelings."""
|
||||
cdef:
|
||||
float64_t emi = 0
|
||||
int64_t n_rows, n_cols
|
||||
float64_t term2, term3, gln
|
||||
int64_t[::1] a_view, b_view
|
||||
float64_t[::1] term1
|
||||
float64_t[::1] gln_a, gln_b, gln_Na, gln_Nb, gln_Nnij, log_Nnij
|
||||
float64_t[::1] log_a, log_b
|
||||
Py_ssize_t i, j, nij
|
||||
int64_t start, end
|
||||
|
||||
n_rows, n_cols = contingency.shape
|
||||
a = np.ravel(contingency.sum(axis=1).astype(np.int64, copy=False))
|
||||
b = np.ravel(contingency.sum(axis=0).astype(np.int64, copy=False))
|
||||
a_view = a
|
||||
b_view = b
|
||||
|
||||
# any labelling with zero entropy implies EMI = 0
|
||||
if a.size == 1 or b.size == 1:
|
||||
return 0.0
|
||||
|
||||
# There are three major terms to the EMI equation, which are multiplied to
|
||||
# and then summed over varying nij values.
|
||||
# While nijs[0] will never be used, having it simplifies the indexing.
|
||||
nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype='float')
|
||||
nijs[0] = 1 # Stops divide by zero warnings. As its not used, no issue.
|
||||
# term1 is nij / N
|
||||
term1 = nijs / n_samples
|
||||
# term2 is log((N*nij) / (a * b)) == log(N * nij) - log(a * b)
|
||||
log_a = np.log(a)
|
||||
log_b = np.log(b)
|
||||
# term2 uses log(N * nij) = log(N) + log(nij)
|
||||
log_Nnij = np.log(n_samples) + np.log(nijs)
|
||||
# term3 is large, and involved many factorials. Calculate these in log
|
||||
# space to stop overflows.
|
||||
gln_a = gammaln(a + 1)
|
||||
gln_b = gammaln(b + 1)
|
||||
gln_Na = gammaln(n_samples - a + 1)
|
||||
gln_Nb = gammaln(n_samples - b + 1)
|
||||
gln_Nnij = gammaln(nijs + 1) + gammaln(n_samples + 1)
|
||||
|
||||
# emi itself is a summation over the various values.
|
||||
for i in range(n_rows):
|
||||
for j in range(n_cols):
|
||||
start = max(1, a_view[i] - n_samples + b_view[j])
|
||||
end = min(a_view[i], b_view[j]) + 1
|
||||
for nij in range(start, end):
|
||||
term2 = log_Nnij[nij] - log_a[i] - log_b[j]
|
||||
# Numerators are positive, denominators are negative.
|
||||
gln = (gln_a[i] + gln_b[j] + gln_Na[i] + gln_Nb[j]
|
||||
- gln_Nnij[nij] - lgamma(a_view[i] - nij + 1)
|
||||
- lgamma(b_view[j] - nij + 1)
|
||||
- lgamma(n_samples - a_view[i] - b_view[j] + nij + 1))
|
||||
term3 = exp(gln)
|
||||
emi += (term1[nij] * term2 * term3)
|
||||
return emi
|
||||
# Authors: The scikit-learn developers
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
from libc.math cimport exp, lgamma
|
||||
|
||||
from ...utils._typedefs cimport float64_t, int64_t
|
||||
|
||||
import numpy as np
|
||||
from scipy.special import gammaln
|
||||
|
||||
|
||||
def expected_mutual_information(contingency, int64_t n_samples):
|
||||
"""Calculate the expected mutual information for two labelings."""
|
||||
cdef:
|
||||
float64_t emi = 0
|
||||
int64_t n_rows, n_cols
|
||||
float64_t term2, term3, gln
|
||||
int64_t[::1] a_view, b_view
|
||||
float64_t[::1] term1
|
||||
float64_t[::1] gln_a, gln_b, gln_Na, gln_Nb, gln_Nnij, log_Nnij
|
||||
float64_t[::1] log_a, log_b
|
||||
Py_ssize_t i, j, nij
|
||||
int64_t start, end
|
||||
|
||||
n_rows, n_cols = contingency.shape
|
||||
a = np.ravel(contingency.sum(axis=1).astype(np.int64, copy=False))
|
||||
b = np.ravel(contingency.sum(axis=0).astype(np.int64, copy=False))
|
||||
a_view = a
|
||||
b_view = b
|
||||
|
||||
# any labelling with zero entropy implies EMI = 0
|
||||
if a.size == 1 or b.size == 1:
|
||||
return 0.0
|
||||
|
||||
# There are three major terms to the EMI equation, which are multiplied to
|
||||
# and then summed over varying nij values.
|
||||
# While nijs[0] will never be used, having it simplifies the indexing.
|
||||
nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype='float')
|
||||
nijs[0] = 1 # Stops divide by zero warnings. As its not used, no issue.
|
||||
# term1 is nij / N
|
||||
term1 = nijs / n_samples
|
||||
# term2 is log((N*nij) / (a * b)) == log(N * nij) - log(a * b)
|
||||
log_a = np.log(a)
|
||||
log_b = np.log(b)
|
||||
# term2 uses log(N * nij) = log(N) + log(nij)
|
||||
log_Nnij = np.log(n_samples) + np.log(nijs)
|
||||
# term3 is large, and involved many factorials. Calculate these in log
|
||||
# space to stop overflows.
|
||||
gln_a = gammaln(a + 1)
|
||||
gln_b = gammaln(b + 1)
|
||||
gln_Na = gammaln(n_samples - a + 1)
|
||||
gln_Nb = gammaln(n_samples - b + 1)
|
||||
gln_Nnij = gammaln(nijs + 1) + gammaln(n_samples + 1)
|
||||
|
||||
# emi itself is a summation over the various values.
|
||||
for i in range(n_rows):
|
||||
for j in range(n_cols):
|
||||
start = max(1, a_view[i] - n_samples + b_view[j])
|
||||
end = min(a_view[i], b_view[j]) + 1
|
||||
for nij in range(start, end):
|
||||
term2 = log_Nnij[nij] - log_a[i] - log_b[j]
|
||||
# Numerators are positive, denominators are negative.
|
||||
gln = (gln_a[i] + gln_b[j] + gln_Na[i] + gln_Nb[j]
|
||||
- gln_Nnij[nij] - lgamma(a_view[i] - nij + 1)
|
||||
- lgamma(b_view[j] - nij + 1)
|
||||
- lgamma(n_samples - a_view[i] - b_view[j] + nij + 1))
|
||||
term3 = exp(gln)
|
||||
emi += (term1[nij] * term2 * term3)
|
||||
return emi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
py.extension_module(
|
||||
'_expected_mutual_info_fast',
|
||||
cython_gen.process('_expected_mutual_info_fast.pyx'),
|
||||
subdir: 'sklearn/metrics/cluster',
|
||||
install: true
|
||||
)
|
||||
py.extension_module(
|
||||
'_expected_mutual_info_fast',
|
||||
cython_gen.process('_expected_mutual_info_fast.pyx'),
|
||||
subdir: 'sklearn/metrics/cluster',
|
||||
install: true
|
||||
)
|
||||
|
||||
@@ -1,49 +1,49 @@
|
||||
# Metrics is cimported from other subpackages so this is needed for the cimport
|
||||
# to work
|
||||
metrics_cython_tree = [
|
||||
fs.copyfile('__init__.py')
|
||||
]
|
||||
# Some metrics code cimports code from utils, we may as well copy all the necessary files
|
||||
metrics_cython_tree += utils_cython_tree
|
||||
|
||||
_dist_metrics_pxd = custom_target(
|
||||
'_dist_metrics_pxd',
|
||||
output: '_dist_metrics.pxd',
|
||||
input: '_dist_metrics.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# Need to install the generated pxd because it is needed in other subpackages
|
||||
# Cython code, e.g. sklearn.cluster
|
||||
install_dir: sklearn_dir / 'metrics',
|
||||
install: true,
|
||||
)
|
||||
metrics_cython_tree += [_dist_metrics_pxd]
|
||||
|
||||
_dist_metrics_pyx = custom_target(
|
||||
'_dist_metrics_pyx',
|
||||
output: '_dist_metrics.pyx',
|
||||
input: '_dist_metrics.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: metrics_cython_tree,
|
||||
)
|
||||
|
||||
_dist_metrics = py.extension_module(
|
||||
'_dist_metrics',
|
||||
cython_gen.process(_dist_metrics_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics',
|
||||
install: true
|
||||
)
|
||||
|
||||
py.extension_module(
|
||||
'_pairwise_fast',
|
||||
[cython_gen.process('_pairwise_fast.pyx'), metrics_cython_tree],
|
||||
dependencies: [openmp_dep],
|
||||
subdir: 'sklearn/metrics',
|
||||
install: true
|
||||
)
|
||||
|
||||
subdir('_pairwise_distances_reduction')
|
||||
subdir('cluster')
|
||||
# Metrics is cimported from other subpackages so this is needed for the cimport
|
||||
# to work
|
||||
metrics_cython_tree = [
|
||||
fs.copyfile('__init__.py')
|
||||
]
|
||||
# Some metrics code cimports code from utils, we may as well copy all the necessary files
|
||||
metrics_cython_tree += utils_cython_tree
|
||||
|
||||
_dist_metrics_pxd = custom_target(
|
||||
'_dist_metrics_pxd',
|
||||
output: '_dist_metrics.pxd',
|
||||
input: '_dist_metrics.pxd.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# Need to install the generated pxd because it is needed in other subpackages
|
||||
# Cython code, e.g. sklearn.cluster
|
||||
install_dir: sklearn_dir / 'metrics',
|
||||
install: true,
|
||||
)
|
||||
metrics_cython_tree += [_dist_metrics_pxd]
|
||||
|
||||
_dist_metrics_pyx = custom_target(
|
||||
'_dist_metrics_pyx',
|
||||
output: '_dist_metrics.pyx',
|
||||
input: '_dist_metrics.pyx.tp',
|
||||
command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
|
||||
# TODO in principle this should go in py.exension_module below. This is
|
||||
# temporary work-around for dependency issue with .pyx.tp files. For more
|
||||
# details, see https://github.com/mesonbuild/meson/issues/13212
|
||||
depends: metrics_cython_tree,
|
||||
)
|
||||
|
||||
_dist_metrics = py.extension_module(
|
||||
'_dist_metrics',
|
||||
cython_gen.process(_dist_metrics_pyx),
|
||||
dependencies: [np_dep],
|
||||
subdir: 'sklearn/metrics',
|
||||
install: true
|
||||
)
|
||||
|
||||
py.extension_module(
|
||||
'_pairwise_fast',
|
||||
[cython_gen.process('_pairwise_fast.pyx'), metrics_cython_tree],
|
||||
dependencies: [openmp_dep],
|
||||
subdir: 'sklearn/metrics',
|
||||
install: true
|
||||
)
|
||||
|
||||
subdir('_pairwise_distances_reduction')
|
||||
subdir('cluster')
|
||||
|
||||
Reference in New Issue
Block a user