add tests for cross_entropy_loss backward pass
finite differences regularly results in estimated gradient of zero, despite the backward pass giving non zero gradient. _probably_ the finite differences fails due to numerical issues
This commit is contained in:
parent
71aaf8dedf
commit
f056a04a80
1 changed files with 19 additions and 0 deletions
|
@ -1090,6 +1090,25 @@ int main(int argc, const char ** argv) {
|
|||
}
|
||||
}
|
||||
|
||||
// cross_entropy_loss
|
||||
{
|
||||
const int nargs = 1;
|
||||
|
||||
int64_t ne2[4];
|
||||
get_random_dims(ne2, 4);
|
||||
|
||||
for (int ndims = 1; ndims <= 3; ++ndims) {
|
||||
x[0] = get_random_tensor(ctx0, ndims, ne2, -1.0f, 1.0f);
|
||||
x[1] = get_random_tensor(ctx0, ndims, ne2, 0.0f, 1.0f);
|
||||
ggml_set_param(ctx0, x[0]);
|
||||
|
||||
struct ggml_tensor * f = ggml_sum(ctx0, ggml_cross_entropy_loss(ctx0, x[0], x[1]));
|
||||
|
||||
check_gradient("cross_entropy_loss", ctx0, x, f, ndims, nargs, 1e-1f, 1e-2f, INFINITY);
|
||||
// finite differences regularly fails!
|
||||
}
|
||||
}
|
||||
|
||||
// rope
|
||||
{
|
||||
const int nargs = 1;
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue