add tests for cross_entropy_loss backward pass
finite differences regularly results in estimated gradient of zero, despite the backward pass giving non zero gradient. _probably_ the finite differences fails due to numerical issues
This commit is contained in:
parent
71aaf8dedf
commit
f056a04a80
1 changed files with 19 additions and 0 deletions
|
@ -1090,6 +1090,25 @@ int main(int argc, const char ** argv) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cross_entropy_loss
|
||||||
|
{
|
||||||
|
const int nargs = 1;
|
||||||
|
|
||||||
|
int64_t ne2[4];
|
||||||
|
get_random_dims(ne2, 4);
|
||||||
|
|
||||||
|
for (int ndims = 1; ndims <= 3; ++ndims) {
|
||||||
|
x[0] = get_random_tensor(ctx0, ndims, ne2, -1.0f, 1.0f);
|
||||||
|
x[1] = get_random_tensor(ctx0, ndims, ne2, 0.0f, 1.0f);
|
||||||
|
ggml_set_param(ctx0, x[0]);
|
||||||
|
|
||||||
|
struct ggml_tensor * f = ggml_sum(ctx0, ggml_cross_entropy_loss(ctx0, x[0], x[1]));
|
||||||
|
|
||||||
|
check_gradient("cross_entropy_loss", ctx0, x, f, ndims, nargs, 1e-1f, 1e-2f, INFINITY);
|
||||||
|
// finite differences regularly fails!
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// rope
|
// rope
|
||||||
{
|
{
|
||||||
const int nargs = 1;
|
const int nargs = 1;
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue